Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.44641852 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.03973412 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.042190623 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1193 1194 1195 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.012654277Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.0491836Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.050770842Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.051014324Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.051063151Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.051133951Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198} {"msg":"Crucible 3a31ced9-6bee-47ca-8319-c89d016713bb has session id: 8430297f-d4d9-47c3-97a1-97b92a7f4854","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.052811683Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"Upstairs opts: Upstairs UUID: 3a31ced9-6bee-47ca-8319-c89d016713bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.052940859Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"Crucible stats registered with UUID: 3a31ced9-6bee-47ca-8319-c89d016713bb","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.052997581Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:24.05304737Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.054650123Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.054804338Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.054849197Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.054943474Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"task":"crutest"} {"msg":"3a31ced9-6bee-47ca-8319-c89d016713bb active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.055212406Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.05523978Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"client":"0","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.055322965Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"client":"1","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.055353325Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.055372423Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"client":"2","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.055461289Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.055634021Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.055691535Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"ds_connection connected from Ok(127.0.0.1:63125)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.056043902Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"ds_connection connected from Ok(127.0.0.1:43781)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.056096627Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"ds_connection connected from Ok(127.0.0.1:37856)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.056135263Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.060322373Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"client":"2","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.060382021Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"client":"0","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.060426359Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"client":"1","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.061828445Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.061912371Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.061939375Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"downstairs","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.061968522Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"downstairs","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.061998983Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"downstairs","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062026607Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"downstairs","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.06205331Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"downstairs","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062082638Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"downstairs","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062109391Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"downstairs","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062136164Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"downstairs","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062165372Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"downstairs","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.06218962Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"downstairs","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.06221486Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"downstairs","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062238787Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"downstairs","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062262635Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"":"downstairs","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062299958Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062323315Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"client":"0","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062348465Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"client":"1","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"Transition from WaitQuorum to Active","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2026-03-04T01:07:24.062380578Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"client":"2","":"downstairs","session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062455396Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062480456Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"3a31ced9-6bee-47ca-8319-c89d016713bb is now active with session: 8430297f-d4d9-47c3-97a1-97b92a7f4854","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062511608Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062534032Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198,"session_id":"8430297f-d4d9-47c3-97a1-97b92a7f4854"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.062557729Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198} {"msg":"Activated sub_volume 3a31ced9-6bee-47ca-8319-c89d016713bb","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:24.06257855Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_live_repair/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:28.211216816Z","hostname":"bmat-EVT22200007-0000ead9","pid":1198} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.264141851Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:28.287667879Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.287753368Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.308514462Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.309176954Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.309251452Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.309275048Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.309310248Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200} {"msg":"Crucible 9bef4d9f-59a5-400e-a0b5-6ecc55bfe0fe has session id: 550b9de4-f452-4bf8-abbe-8c49e7ac43ef","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.310395837Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"Upstairs opts: Upstairs UUID: 9bef4d9f-59a5-400e-a0b5-6ecc55bfe0fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.31054849Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"Crucible stats registered with UUID: 9bef4d9f-59a5-400e-a0b5-6ecc55bfe0fe","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.310593469Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:28.310630322Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.31088208Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.310949704Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.31098794Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.311256171Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.311548419Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200} {"msg":"9bef4d9f-59a5-400e-a0b5-6ecc55bfe0fe active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.311641273Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.311736642Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"client":"0","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.31177104Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"client":"1","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.311798945Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"client":"2","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.311850137Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.312002849Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.312036165Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:53895)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.312161693Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"io task","client":"2","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:59362)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.312266541Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"io task","client":"1","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:61138)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.312505755Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"io task","client":"0","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.313896949Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"client":"1","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.314052226Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"client":"2","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.314530413Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"client":"0","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.315106764Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.315144249Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.315949723Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"downstairs","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316178967Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"downstairs","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316232162Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"downstairs","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316275117Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"downstairs","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.31631676Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"downstairs","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316364134Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"downstairs","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316409985Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"downstairs","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316450765Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"downstairs","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316494522Z","hostname":"Wait for a query_work_queue command to finish before sending IO bmat-EVT22200007-0000ead9","pid":1200,"":"downstairs","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316541936Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"downstairs","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316573258Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"downstairs","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316606293Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"downstairs","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.31664494Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"":"downstairs","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316688466Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316722203Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"client":"0","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316757332Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"client":"1","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316792552Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"client":"2","":"downstairs","session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316829104Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316865005Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"9bef4d9f-59a5-400e-a0b5-6ecc55bfe0fe is now active with session: 550b9de4-f452-4bf8-abbe-8c49e7ac43ef","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316901377Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316930826Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200,"session_id":"550b9de4-f452-4bf8-abbe-8c49e7ac43ef"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:28.316962829Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200} {"msg":"Activated sub_volume 9bef4d9f-59a5-400e-a0b5-6ecc55bfe0fe","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-04T01:07:28.316991586Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 451 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Write block 541 len 12288 data: 2 2 2 03/30 Read block 164 len 40960 04/30 Read block 424 len 20480 05/30 Read block 39 len 36864 06/30 Read block 583 len 12288 07/30 Read block 578 len 32768 08/30 Read block 401 len 4096 09/30 Read block 309 len 32768 10/30 Read block 575 len 40960 11/30 Write block 176 len 8192 data: 2 2 12/30 Read block 209 len 28672 13/30 Write block 592 len 24576 data: 2 2 2 2 2 2 14/30 Read block 392 len 40960 15/30 Write block 454 len 28672 data: 3 3 3 3 3 2 2 16/30 Flush 17/30 Write block 508 len 40960 data: 2 2 2 2 2 2 2 2 2 2 18/30 Read block 278 len 20480 19/30 Read block 249 len 8192 20/30 Read block 126 len 12288 21/30 Flush 22/30 Write block 381 len 8192 data: 2 2 23/30 Read block 580 len 36864 24/30 Write block 540 len 4096 data: 2 25/30 Flush 26/30 Write block 252 len 16384 data: 2 2 2 2 27/30 Write block 134 len 36864 data: 2 2 2 2 2 2 2 2 2 28/30 Read block 337 len 16384 29/30 Write block 415 len 12288 data: 2 2 2 30/30 Write block 64 len 8192 data: 2 2 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Sent Done Done false 1001 Acked Write 3 Sent Done Done false 1002 Acked Read 10 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:28.35074347Z","hostname":"bmat-EVT22200007-0000ead9","pid":1200} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F T T <--- 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F T T <--- 7 140-159 1 1 1 1 1 1 F T T <--- 8 160-179 1 2 2 1 2 2 F F F <--- 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F T T <--- 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 2 2 1 4 4 F F F <--- 20 400-419 1 1 1 1 1 1 F T T <--- 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 2 2 1 2 2 F F F <--- 23 460-479 1 2 2 1 2 2 F F F <--- 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 2 2 1 3 3 F F F <--- 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 2 2 1 4 4 F F F <--- 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 2 2 1 2 2 F F F <--- Max gen: 2, Max flush: 4 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 3 3 3 5 5 5 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 3 3 3 5 5 5 F F F 7 140-159 3 3 3 5 5 5 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 3 3 3 5 5 5 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 3 3 3 5 5 5 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 3 3 3 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 4 4 4 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 2 2 2 2 2 2 F F F Max gen: 3, Max flush: 5 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.372851198Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.4047689Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.405538354Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.405649835Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.405698531Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.405763149Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208} {"msg":"Crucible 7f0acf91-e805-424b-9357-5eddcb8ab905 has session id: b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.407076018Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"Upstairs opts: Upstairs UUID: 7f0acf91-e805-424b-9357-5eddcb8ab905, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.407175595Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"Crucible stats registered with UUID: 7f0acf91-e805-424b-9357-5eddcb8ab905","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.407213701Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:33.407241907Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.407888517Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.407948876Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.407980399Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.408213841Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.408247548Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208} {"msg":"7f0acf91-e805-424b-9357-5eddcb8ab905 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.408522522Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.408554886Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"client":"0","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.408584695Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"client":"1","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.408614745Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"client":"2","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.408643892Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.408674393Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.408752988Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"ds_connection connected from Ok(127.0.0.1:59222)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.408875892Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"ds_connection connected from Ok(127.0.0.1:53915)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.408912334Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"ds_connection connected from Ok(127.0.0.1:51366)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.409247237Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.411076108Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"client":"0","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.411154393Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"client":"2","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.411792266Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"client":"1","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.411885631Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.411928576Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 5, 1, 1, 5, 5, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.412719642Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"downstairs","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 3, 1, 1, 3, 3, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.412778909Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"downstairs","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.412843697Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"downstairs","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 5, 1, 1, 5, 5, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.412891813Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"downstairs","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 3, 1, 1, 3, 3, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.412938976Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"downstairs","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.412991019Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"downstairs","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 5, 1, 1, 5, 5, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413038433Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"downstairs","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 3, 1, 1, 3, 3, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413099123Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"downstairs","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413155164Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"downstairs","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413204652Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"downstairs","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413249601Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"downstairs","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413291143Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"downstairs","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413333197Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"":"downstairs","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413377534Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:07:33.413423676Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"client":"0","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413488083Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"client":"1","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413532181Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"client":"2","":"downstairs","session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413586077Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413628622Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"7f0acf91-e805-424b-9357-5eddcb8ab905 is now active with session: b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413676226Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413720904Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208,"session_id":"b7a43b62-cd0a-4d58-b2f1-b21ce28c3d39"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.41376907Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208} {"msg":"Activated sub_volume 7f0acf91-e805-424b-9357-5eddcb8ab905","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:33.413806965Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 355 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Write block 294 len 16384 data: 2 2 2 2 03/30 Read block 4 len 28672 04/30 Write block 496 len 16384 data: 2 2 2 2 05/30 Read block 221 len 24576 06/30 Write block 590 len 32768 data: 2 2 3 3 3 3 3 3 07/30 Read block 515 len 16384 08/30 Read block 3 len 12288 09/30 Read block 411 len 20480 10/30 Read block 565 len 40960 11/30 Read block 509 len 12288 12/30 Flush 13/30 Flush 14/30 Write block 209 len 16384 data: 2 2 2 2 15/30 Read block 47 len 4096 16/30 Read block 337 len 24576 17/30 Write block 2 len 32768 data: 2 2 2 2 2 2 2 2 18/30 Read block 312 len 40960 19/30 Read block 586 len 32768 20/30 Read block 181 len 32768 21/30 Write block 145 len 28672 data: 2 2 2 2 2 2 2 22/30 Read block 83 len 16384 23/30 Read block 588 len 20480 24/30 Read block 550 len 24576 25/30 Write block 510 len 16384 data: 3 3 3 3 26/30 Write block 87 len 32768 data: 2 2 2 2 2 2 2 2 27/30 Write block 429 len 16384 data: 2 2 2 2 28/30 Read block 367 len 12288 29/30 Read block 587 len 12288 30/30 Read block 218 len 32768 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Sent Done false 1001 Acked Write 4 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Write 7 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 0 31 Done 29 0 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) None Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:33.454713143Z","hostname":"bmat-EVT22200007-0000ead9","pid":1208} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 T F T <--- 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 3 3 3 5 5 5 F F F 4 080-099 1 1 1 1 1 1 T F T <--- 5 100-119 1 1 1 1 1 1 F F F 6 120-139 3 3 3 5 5 5 F F F 7 140-159 3 3 3 5 5 5 T F T <--- 8 160-179 2 2 2 2 2 2 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 T F T <--- 11 220-239 1 1 1 1 1 1 F F F 12 240-259 3 3 3 5 5 5 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 1 4 6 1 6 F F F <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 1 4 6 1 6 F F F <--- 18 360-379 4 1 4 6 1 6 F F F <--- 19 380-399 2 2 2 4 4 4 F F F 20 400-419 3 3 3 5 5 5 F F F 21 420-439 1 1 1 1 1 1 T F T <--- 22 440-459 2 2 2 2 2 2 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 4 1 4 6 1 6 F F F <--- 25 500-519 2 2 2 3 3 3 T F T <--- 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 4 4 4 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 2 4 6 2 6 F F F <--- Max gen: 4, Max flush: 6 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 5 5 5 7 7 7 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 3 3 3 5 5 5 F F F 4 080-099 5 5 5 7 7 7 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 3 3 3 5 5 5 F F F 7 140-159 5 5 5 7 7 7 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 5 5 5 7 7 7 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 3 3 3 5 5 5 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 6 6 6 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 6 6 6 F F F 18 360-379 4 4 4 6 6 6 F F F 19 380-399 2 2 2 4 4 4 F F F 20 400-419 3 3 3 5 5 5 F F F 21 420-439 5 5 5 7 7 7 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 2 2 2 2 2 2 F F F 24 480-499 4 4 4 6 6 6 F F F 25 500-519 5 5 5 7 7 7 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 4 4 4 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 6 6 6 F F F Max gen: 5, Max flush: 7 Begin loop 3 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.182032369Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.218188483Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.21891925Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.218999839Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.219025901Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.219061772Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216} {"msg":"Crucible 094c42fe-e94c-425b-8368-6c62d3cfed4c has session id: 4ba71c49-e6db-408f-be84-09e4d704a2ed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.220614155Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Upstairs opts: Upstairs UUID: 094c42fe-e94c-425b-8368-6c62d3cfed4c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.220736367Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Crucible stats registered with UUID: 094c42fe-e94c-425b-8368-6c62d3cfed4c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.22077821Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:38.220821867Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.221166529Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.221254904Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.221297448Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.221718562Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.221766056Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216} {"msg":"094c42fe-e94c-425b-8368-6c62d3cfed4c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.221796546Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.221844161Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"client":"0","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.221883699Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"client":"1","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.221917356Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"client":"2","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.22195003Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.221984448Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.222019508Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:60584)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.222200276Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:41654)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.222254323Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:44136)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.222299141Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.224175637Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"client":"1","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.224365793Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"client":"2","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.225205345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.225262288Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"client":"0","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.225304141Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"[0]R flush_numbers[0..12]: [7, 1, 1, 5, 7, 1, 5, 7, 2, 1, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226385672Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"downstairs","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"[0]R generation[0..12]: [5, 1, 1, 3, 5, 1, 3, 5, 2, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226447515Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"downstairs","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226494397Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"downstairs","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"[1]R flush_numbers[0..12]: [7, 1, 1, 5, 7, 1, 5, 7, 2, 1, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226537032Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"downstairs","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"[1]R generation[0..12]: [5, 1, 1, 3, 5, 1, 3, 5, 2, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226579045Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"downstairs","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226623994Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"downstairs","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"[2]R flush_numbers[0..12]: [7, 1, 1, 5, 7, 1, 5, 7, 2, 1, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226665837Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"downstairs","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R generation[0..12]: [5, 1, 1, 3, 5, 1, 3, 5, 2, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226714914Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"downstairs","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226761456Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"downstairs","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226798019Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"downstairs","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226837176Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"downstairs","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226876314Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"downstairs","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.22691514Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"":"downstairs","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226952705Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.226989648Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 bmat-EVT22200007-0000ead9","pid":1216,"client":"0","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.22703592Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"client":"1","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.227078654Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"client":"2","":"downstairs","session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.227119194Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.227157711Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"094c42fe-e94c-425b-8368-6c62d3cfed4c is now active with session: 4ba71c49-e6db-408f-be84-09e4d704a2ed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.227198822Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.22723326Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216,"session_id":"4ba71c49-e6db-408f-be84-09e4d704a2ed"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.227269863Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216} {"msg":"Activated sub_volume 094c42fe-e94c-425b-8368-6c62d3cfed4c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:38.227302036Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 514 len 4096 data: 3 02/30 Write block 499 len 36864 data: 3 2 2 2 2 2 2 2 2 03/30 Write block 294 len 20480 data: 3 3 3 3 2 04/30 Write block 409 len 12288 data: 2 2 2 05/30 Write block 554 len 12288 data: 2 2 2 06/30 Write block 518 len 36864 data: 2 2 2 2 2 2 2 2 2 07/30 Read block 54 len 28672 08/30 Write block 146 len 12288 data: 3 3 3 09/30 Read block 155 len 12288 10/30 Flush 11/30 Write block 33 len 36864 data: 2 2 2 2 2 2 2 2 2 12/30 Read block 495 len 20480 13/30 Read block 335 len 28672 14/30 Read block 20 len 36864 15/30 Read block 533 len 28672 16/30 Write block 472 len 40960 data: 2 2 2 2 2 2 2 2 2 2 17/30 Write block 316 len 36864 data: 2 2 2 2 2 2 2 2 2 18/30 Flush 19/30 Write block 147 len 32768 data: 4 4 3 3 3 2 2 2 20/30 Write block 47 len 32768 data: 2 2 2 2 2 2 2 2 21/30 Write block 13 len 40960 data: 2 2 2 2 2 2 2 2 2 2 22/30 Read block 291 len 24576 23/30 Write block 311 len 4096 data: 2 24/30 Read block 532 len 8192 25/30 Write block 346 len 24576 data: 2 2 2 2 2 2 26/30 Read block 100 len 24576 27/30 Write block 397 len 20480 data: 2 2 2 2 2 28/30 Write block 346 len 32768 data: 3 3 3 3 3 3 2 2 29/30 Read block 146 len 40960 30/30 Write block 255 len 20480 data: 3 2 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 9 Sent Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Write 3 Sent Done Done false 1004 Acked Write 3 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:38.260080273Z","hostname":"bmat-EVT22200007-0000ead9","pid":1216} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 5 5 5 7 7 7 F T T <--- 1 020-039 1 6 6 1 9 9 F T T <--- 2 040-059 1 6 6 1 9 9 F T T <--- 3 060-079 3 3 3 5 5 5 F F F 4 080-099 5 5 5 7 7 7 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 3 3 3 5 5 5 F F F 7 140-159 5 6 6 7 8 8 F T T <--- 8 160-179 2 2 2 2 2 2 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 5 5 5 7 7 7 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 3 3 3 5 5 5 F T T <--- 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 6 6 6 8 8 F F F <--- 15 300-319 1 6 6 1 9 9 F T T <--- 16 320-339 1 6 6 1 9 9 F F F <--- 17 340-359 4 4 4 6 6 6 F T T <--- 18 360-379 4 4 4 6 6 6 F F F 19 380-399 2 2 2 4 4 4 F T T <--- 20 400-419 3 6 6 5 8 8 F T T <--- 21 420-439 5 5 5 7 7 7 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 2 6 6 2 9 9 F F F <--- 24 480-499 4 6 6 6 9 9 F F F <--- 25 500-519 5 6 6 7 8 8 T F F <--- 26 520-539 1 6 6 1 8 8 F F F <--- 27 540-559 2 6 6 4 8 8 F F F <--- 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 6 6 6 F F F Max gen: 6, Max flush: 9 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 10 10 10 F F F 1 020-039 7 7 7 10 10 10 F F F 2 040-059 7 7 7 10 10 10 F F F 3 060-079 3 3 3 5 5 5 F F F 4 080-099 5 5 5 7 7 7 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 3 3 3 5 5 5 F F F 7 140-159 7 7 7 10 10 10 F F F 8 160-179 2 2 2 2 2 2 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 5 5 5 7 7 7 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 7 7 7 10 10 10 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 6 6 6 8 8 8 F F F 15 300-319 7 7 7 10 10 10 F F F 16 320-339 6 6 6 9 9 9 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 4 4 4 6 6 6 F F F 19 380-399 7 7 7 10 10 10 F F F 20 400-419 7 7 7 10 10 10 F F F 21 420-439 5 5 5 7 7 7 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 6 6 6 9 9 9 F F F 24 480-499 6 6 6 9 9 9 F F F 25 500-519 6 6 6 8 8 8 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 6 6 6 8 8 8 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 6 6 6 F F F Max gen: 7, Max flush: 10 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.93675505Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.964061653Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.964628586Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.964743423Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.964769054Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.964805175Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224} {"msg":"Crucible 63d2a4d5-3f6e-4195-8a0a-1a877eb3a70d has session id: 38ba422f-68e6-4223-b05c-27f78f510936","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.966042615Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"Upstairs opts: Upstairs UUID: 63d2a4d5-3f6e-4195-8a0a-1a877eb3a70d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.966116702Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"Crucible stats registered with UUID: 63d2a4d5-3f6e-4195-8a0a-1a877eb3a70d","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.966153705Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:42.966179426Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.966436926Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.966504199Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.966624487Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.967077615Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.967106983Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224} {"msg":"63d2a4d5-3f6e-4195-8a0a-1a877eb3a70d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.967215688Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.967246409Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"client":"0","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.967273843Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"client":"1","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.967301107Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"client":"2","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.967328381Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.967356026Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.967521433Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:42.967809614Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:42.967870204Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:42.967912447Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"ds_connection connected from Ok(127.0.0.1:48711)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.968106472Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.968144407Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"ds_connection connected from Ok(127.0.0.1:62468)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.968184366Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.968213493Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.969921826Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"client":"0","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.970208613Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"client":"2","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.970765145Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:42.971102443Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.970227116Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"ds_connection connected from Ok(127.0.0.1:40258)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.970579614Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.97285586Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"client":"1","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"[0]R flush_numbers[0..12]: [10, 10, 10, 5, 7, 1, 5, 10, 2, 1, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974187026Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"downstairs","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"[0]R generation[0..12]: [7, 7, 7, 3, 5, 1, 3, 7, 2, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974231935Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"downstairs","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974279689Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"downstairs","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"[1]R flush_numbers[0..12]: [10, 10, 10, 5, 7, 1, 5, 10, 2, 1, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974312885Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"downstairs","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"[1]R generation[0..12]: [7, 7, 7, 3, 5, 1, 3, 7, 2, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974344578Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"downstairs","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974379637Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"downstairs","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"[2]R flush_numbers[0..12]: [10, 10, 10, 5, 7, 1, 5, 10, 2, 1, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974412182Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"downstairs","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"[2]R generation[0..12]: [7, 7, 7, 3, 5, 1, 3, 7, 2, 1, 5, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974443784Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"downstairs","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974478874Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"downstairs","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.97450754Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"downstairs","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.97453753Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"downstairs","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974565435Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"downstairs","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974593751Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"":"downstairs","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.97462329Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974650013Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"client":"0","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974679541Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"client":"1","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974708769Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"client":"2","":"downstairs","session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974739069Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974770481Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"63d2a4d5-3f6e-4195-8a0a-1a877eb3a70d is now active with session: 38ba422f-68e6-4223-b05c-27f78f510936","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974802545Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974829207Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224,"session_id":"38ba422f-68e6-4223-b05c-27f78f510936"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974857083Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224} {"msg":"Activated sub_volume 63d2a4d5-3f6e-4195-8a0a-1a877eb3a70d","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:52.974881511Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 299 len 12288 data: 2 2 2 02/30 Read block 211 len 32768 03/30 Read block 402 len 12288 04/30 Write block 11 len 20480 data: 2 2 3 3 3 05/30 Read block 388 len 32768 06/30 Write block 272 len 12288 data: 2 2 2 07/30 Write block 201 len 32768 data: 2 2 2 2 2 2 2 2 08/30 Write block 141 len 12288 data: 3 3 2 09/30 Flush 10/30 Write block 160 len 4096 data: 2 11/30 Read block 330 len 28672 12/30 Read block 290 len 20480 13/30 Write block 310 len 12288 data: 2 3 2 14/30 Read block 323 len 20480 15/30 Read block 417 len 16384 16/30 Write block 249 len 8192 data: 2 2 17/30 Read block 334 len 24576 18/30 Write block 186 len 36864 data: 2 2 2 2 2 2 2 2 2 19/30 Write block 390 len 20480 data: 2 2 2 2 2 20/30 Flush 21/30 Write block 424 len 24576 data: 2 2 2 2 2 3 22/30 Write block 196 len 32768 data: 2 2 2 2 2 3 3 3 23/30 Write block 27 len 4096 data: 2 24/30 Read block 323 len 4096 25/30 Write block 64 len 4096 data: 3 26/30 Write block 16 len 36864 data: 3 3 3 3 3 3 3 2 2 27/30 Read block 49 len 8192 28/30 Write block 193 len 32768 data: 3 3 2 3 3 3 3 3 29/30 Write block 216 len 12288 data: 2 2 2 30/30 Write block 541 len 16384 data: 3 3 3 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 22 3 28 Done 27 8 27 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:53.008244695Z","hostname":"bmat-EVT22200007-0000ead9","pid":1224} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 8 7 8 11 10 11 T T T <--- 1 020-039 7 7 7 10 10 10 T F T <--- 2 040-059 7 7 7 10 10 10 F F F 3 060-079 3 3 3 5 5 5 T F T <--- 4 080-099 5 5 5 7 7 7 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 3 3 3 5 5 5 F F F 7 140-159 8 7 8 11 10 11 F T F <--- 8 160-179 8 2 8 12 2 12 F F F <--- 9 180-199 8 1 8 12 1 12 T F T <--- 10 200-219 8 5 8 11 7 11 T T T <--- 11 220-239 1 1 1 1 1 1 F F F 12 240-259 8 7 8 12 10 12 F F F <--- 13 260-279 8 1 8 11 1 11 F T F <--- 14 280-299 8 6 8 11 8 11 F T F <--- 15 300-319 8 7 8 12 10 12 F T F <--- 16 320-339 6 6 6 9 9 9 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 4 4 4 6 6 6 F F F 19 380-399 8 7 8 12 10 12 F F F <--- 20 400-419 7 7 7 10 10 10 F F F 21 420-439 5 5 5 7 7 7 T F T <--- 22 440-459 2 2 2 2 2 2 F F F 23 460-479 6 6 6 9 9 9 F F F 24 480-499 6 6 6 9 9 9 F F F 25 500-519 6 6 6 8 8 8 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 6 6 6 8 8 8 T F T <--- 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 6 6 6 F F F Max gen: 8, Max flush: 12 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 9 9 9 13 13 13 F F F 2 040-059 7 7 7 10 10 10 F F F 3 060-079 9 9 9 13 13 13 F F F 4 080-099 5 5 5 7 7 7 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 3 3 3 5 5 5 F F F 7 140-159 8 8 8 11 11 11 F F F 8 160-179 8 8 8 12 12 12 F F F 9 180-199 9 9 9 13 13 13 F F F 10 200-219 9 9 9 13 13 13 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 8 8 8 12 12 12 F F F 13 260-279 8 8 8 11 11 11 F F F 14 280-299 8 8 8 11 11 11 F F F 15 300-319 8 8 8 12 12 12 F F F 16 320-339 6 6 6 9 9 9 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 4 4 4 6 6 6 F F F 19 380-399 8 8 8 12 12 12 F F F 20 400-419 7 7 7 10 10 10 F F F 21 420-439 9 9 9 13 13 13 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 6 6 6 9 9 9 F F F 24 480-499 6 6 6 9 9 9 F F F 25 500-519 6 6 6 8 8 8 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 9 9 9 13 13 13 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 4 4 4 6 6 6 F F F Max gen: 9, Max flush: 13 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.654493094Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.686005695Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.686596935Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.686692484Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.686724578Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.686774106Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232} {"msg":"Crucible 533e32dd-a2ae-4c1f-8c3e-bd1501128e2b has session id: 62f495df-6fb6-4fc5-a4c4-c45d45b90262","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.688121313Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Upstairs opts: Upstairs UUID: 533e32dd-a2ae-4c1f-8c3e-bd1501128e2b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.688208626Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Crucible stats registered with UUID: 533e32dd-a2ae-4c1f-8c3e-bd1501128e2b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.688238024Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:57.688269737Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.688702234Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.688790388Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.688818864Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.689326499Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.689428531Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232} {"msg":"533e32dd-a2ae-4c1f-8c3e-bd1501128e2b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.689455254Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.689489021Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"client":"0","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.689516516Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"client":"1","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.689541475Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"client":"2","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.689566244Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.689592656Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.689618087Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"ds_connection connected from Ok(127.0.0.1:38682)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.68978153Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"ds_connection connected from Ok(127.0.0.1:41696)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.689820387Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"ds_connection connected from Ok(127.0.0.1:46146)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.689884244Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.691791901Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"client":"2","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.692058198Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"client":"0","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.692183055Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"client":"1","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.69267042Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.692976135Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"[0]R flush_numbers[0..12]: [13, 13, 10, 13, 7, 1, 5, 11, 12, 13, 13, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693204297Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"downstairs","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"[0]R generation[0..12]: [9, 9, 7, 9, 5, 1, 3, 8, 8, 9, 9, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693265268Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"downstairs","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693320257Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"downstairs","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"[1]R flush_numbers[0..12]: [13, 13, 10, 13, 7, 1, 5, 11, 12, 13, 13, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693369484Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"downstairs","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"[1]R generation[0..12]: [9, 9, 7, 9, 5, 1, 3, 8, 8, 9, 9, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693418772Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"downstairs","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693476075Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"downstairs","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"[2]R flush_numbers[0..12]: [13, 13, 10, 13, 7, 1, 5, 11, 12, 13, 13, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693529441Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"downstairs","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R generation[0..12]: [9, 9, 7, 9, 5, 1, 3, 8, 8, 9, 9, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693594369Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"downstairs","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693656973Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"downstairs","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693706782Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"downstairs","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693768514Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"downstairs","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693815166Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"downstairs","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693856769Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"":"downstairs","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.69390796Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693947729Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"client":"0","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.693985564Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"client":"1","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.694031946Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"client":"2","":"downstairs","session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.69408478Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.694128737Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"533e32dd-a2ae-4c1f-8c3e-bd1501128e2b is now active with session: 62f495df-6fb6-4fc5-a4c4-c45d45b90262","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.694181662Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.694224517Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232,"session_id":"62f495df-6fb6-4fc5-a4c4-c45d45b90262"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.694269336Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232} {"msg":"Activated sub_volume 533e32dd-a2ae-4c1f-8c3e-bd1501128e2b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:07:57.694307231Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 325 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Read block 458 len 20480 03/30 Write block 547 len 24576 data: 2 2 2 2 2 2 04/30 Read block 124 len 20480 05/30 Write block 176 len 28672 data: 3 3 2 2 2 2 2 06/30 Flush 07/30 Write block 201 len 32768 data: 4 4 4 3 3 3 3 3 08/30 Flush 09/30 Write block 243 len 32768 data: 2 2 2 2 2 2 3 3 10/30 Flush 11/30 Write block 165 len 4096 data: 2 12/30 Write block 529 len 24576 data: 2 2 2 2 2 2 13/30 Flush 14/30 Write block 239 len 36864 data: 2 2 2 2 3 3 3 3 3 15/30 Read block 436 len 4096 16/30 Write block 230 len 40960 data: 2 2 2 2 2 2 2 2 2 3 17/30 Flush 18/30 Write block 246 len 28672 data: 4 4 3 4 4 2 3 19/30 Write block 154 len 16384 data: 3 2 2 2 20/30 Read block 136 len 16384 21/30 Read block 222 len 28672 22/30 Write block 557 len 32768 data: 2 2 2 2 2 2 2 2 23/30 Read block 288 len 32768 24/30 Read block 516 len 20480 25/30 Read block 203 len 8192 26/30 Write block 122 len 4096 data: 2 27/30 Read block 276 len 36864 28/30 Write block 260 len 8192 data: 2 2 29/30 Read block 236 len 12288 30/30 Write block 420 len 24576 data: 2 2 2 2 3 3 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Sent Done false 1001 Acked Read 5 Done Sent Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Read 5 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 8 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) None Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:07:57.734800362Z","hostname":"bmat-EVT22200007-0000ead9","pid":1232} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 9 9 9 13 13 13 F F F 2 040-059 7 7 7 10 10 10 F F F 3 060-079 9 9 9 13 13 13 F F F 4 080-099 5 5 5 7 7 7 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 3 3 3 5 5 5 T F T <--- 7 140-159 8 8 8 11 11 11 T F T <--- 8 160-179 10 8 10 17 12 17 F F F <--- 9 180-199 10 9 10 14 13 14 F F F <--- 10 200-219 10 9 10 15 13 15 F F F <--- 11 220-239 10 1 10 18 1 18 F F F <--- 12 240-259 10 8 10 18 12 18 T F T <--- 13 260-279 8 8 8 11 11 11 T F T <--- 14 280-299 8 8 8 11 11 11 F F F 15 300-319 8 8 8 12 12 12 F F F 16 320-339 10 6 10 14 9 14 F F F <--- 17 340-359 7 7 7 10 10 10 F F F 18 360-379 4 4 4 6 6 6 F F F 19 380-399 8 8 8 12 12 12 F F F 20 400-419 7 7 7 10 10 10 F F F 21 420-439 9 9 9 13 13 13 T F T <--- 22 440-459 2 2 2 2 2 2 F F F 23 460-479 6 6 6 9 9 9 F F F 24 480-499 6 6 6 9 9 9 F F F 25 500-519 6 6 6 8 8 8 F F F 26 520-539 10 6 10 17 8 17 F F F <--- 27 540-559 10 9 10 14 13 14 T F T <--- 28 560-579 1 1 1 1 1 1 T F T <--- 29 580-599 4 4 4 6 6 6 F F F Max gen: 10, Max flush: 18 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 9 9 9 13 13 13 F F F 2 040-059 7 7 7 10 10 10 F F F 3 060-079 9 9 9 13 13 13 F F F 4 080-099 5 5 5 7 7 7 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 11 11 11 19 19 19 F F F 7 140-159 11 11 11 19 19 19 F F F 8 160-179 10 10 10 17 17 17 F F F 9 180-199 10 10 10 14 14 14 F F F 10 200-219 10 10 10 15 15 15 F F F 11 220-239 10 10 10 18 18 18 F F F 12 240-259 11 11 11 19 19 19 F F F 13 260-279 11 11 11 19 19 19 F F F 14 280-299 8 8 8 11 11 11 F F F 15 300-319 8 8 8 12 12 12 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 4 4 4 6 6 6 F F F 19 380-399 8 8 8 12 12 12 F F F 20 400-419 7 7 7 10 10 10 F F F 21 420-439 11 11 11 19 19 19 F F F 22 440-459 2 2 2 2 2 2 F F F 23 460-479 6 6 6 9 9 9 F F F 24 480-499 6 6 6 9 9 9 F F F 25 500-519 6 6 6 8 8 8 F F F 26 520-539 10 10 10 17 17 17 F F F 27 540-559 11 11 11 19 19 19 F F F 28 560-579 11 11 11 19 19 19 F F F 29 580-599 4 4 4 6 6 6 F F F Max gen: 11, Max flush: 19 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.37089172Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.418801785Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.419656928Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.419751365Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.419790072Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.419840672Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240} {"msg":"Crucible 37e82e00-6f34-4b38-be6e-fc87464933d1 has session id: 859f4709-4be3-468e-8344-77d3494a7a41","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.420815402Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"Upstairs opts: Upstairs UUID: 37e82e00-6f34-4b38-be6e-fc87464933d1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.420892765Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"Crucible stats registered with UUID: 37e82e00-6f34-4b38-be6e-fc87464933d1","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.420933025Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:02.420967964Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.421341093Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.421406051Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.421445289Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.421675936Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.421712939Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240} {"msg":"37e82e00-6f34-4b38-be6e-fc87464933d1 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.421821193Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.421858307Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"client":"0","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.421896412Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"client":"1","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.421933776Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"client":"2","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.421975859Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.422015067Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.422053583Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"ds_connection connected from Ok(127.0.0.1:57095)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.422383957Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"ds_connection connected from Ok(127.0.0.1:41422)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.422469316Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"ds_connection connected from Ok(127.0.0.1:34029)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.422641407Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.424351383Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"client":"1","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.424579214Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"client":"2","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.424625896Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"client":"0","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.42508242Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.425442643Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"[0]R flush_numbers[0..12]: [13, 13, 10, 13, 7, 1, 19, 19, 17, 14, 15, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.425561308Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"downstairs","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"[0]R generation[0..12]: [9, 9, 7, 9, 5, 1, 11, 11, 10, 10, 10, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.425611367Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"downstairs","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.425670154Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"downstairs","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"[1]R flush_numbers[0..12]: [13, 13, 10, 13, 7, 1, 19, 19, 17, 14, 15, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.425714862Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"downstairs","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"[1]R generation[0..12]: [9, 9, 7, 9, 5, 1, 11, 11, 10, 10, 10, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.425752757Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"downstairs","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.425796424Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"downstairs","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"[2]R flush_numbers[0..12]: [13, 13, 10, 13, 7, 1, 19, 19, 17, 14, 15, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.425845721Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"downstairs","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"[2]R generation[0..12]: [9, 9, 7, 9, 5, 1, 11, 11, 10, 10, 10, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.425893396Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"downstairs","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.425929988Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"downstairs","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.425958384Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"downstairs","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.425988133Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.426034936Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"downstairs","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.426063142Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"":"downstairs","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.426100536Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.426140665Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"client":"0","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.426185514Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"client":"1","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.426216104Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"client":"2","":"downstairs","session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.426246164Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.42627947Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"37e82e00-6f34-4b38-be6e-fc87464933d1 is now active with session: 859f4709-4be3-468e-8344-77d3494a7a41","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.426332715Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.42637546Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240,"session_id":"859f4709-4be3-468e-8344-77d3494a7a41"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:02.426404778Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240} {"msg":"Activated sub_volume 37e82e00-6f34-4b38-be6e-fc87464933d1","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-04T01:08:02.426429146Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 468 len 40960 data: 2 2 2 2 3 3 3 3 3 3 02/30 Read block 76 len 32768 03/30 Read block 20 len 32768 04/30 Read block 537 len 40960 05/30 Write block 191 len 8192 data: 3 3 06/30 Write block 400 len 12288 data: 3 3 2 07/30 Write block 438 len 36864 data: 2 2 2 2 2 2 2 2 2 08/30 Read block 314 len 24576 09/30 Write block 596 len 8192 data: 4 4 10/30 Read block 476 len 24576 11/30 Read block 126 len 20480 12/30 Read block 215 len 12288 13/30 Read block 8 len 16384 14/30 Read block 222 len 16384 15/30 Write block 536 len 28672 data: 2 2 2 2 3 4 4 16/30 Write block 392 len 20480 data: 3 3 3 2 2 17/30 Flush 18/30 Write block 163 len 24576 data: 2 2 3 2 2 2 19/30 Read block 471 len 8192 20/30 Flush 21/30 Write block 478 len 40960 data: 3 3 3 3 2 2 2 2 2 2 22/30 Read block 318 len 8192 23/30 Read block 327 len 16384 24/30 Flush 25/30 Write block 494 len 24576 data: 2 2 3 3 3 4 26/30 Read block 456 len 16384 27/30 Read block 341 len 8192 28/30 Write block 136 len 36864 data: 3 3 3 3 3 4 4 3 2 29/30 Read block 33 len 16384 30/30 Write block 482 len 8192 data: 3 3 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 8 Sent Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Read 10 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Read 4 Sent Sent Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 1 32 Done 1 28 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:02.461202413Z","hostname":"bmat-EVT22200007-0000ead9","pid":1240} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 9 9 9 13 13 13 F F F 2 040-059 7 7 7 10 10 10 F F F 3 060-079 9 9 9 13 13 13 F F F 4 080-099 5 5 5 7 7 7 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 11 11 11 19 19 19 F T T <--- 7 140-159 11 11 11 19 19 19 F T T <--- 8 160-179 10 12 12 17 21 21 F F F <--- 9 180-199 10 12 12 14 20 20 F F F <--- 10 200-219 10 10 10 15 15 15 F F F 11 220-239 10 10 10 18 18 18 F F F 12 240-259 11 11 11 19 19 19 F F F 13 260-279 11 11 11 19 19 19 F F F 14 280-299 8 8 8 11 11 11 F F F 15 300-319 8 8 8 12 12 12 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 4 4 4 6 6 6 F F F 19 380-399 8 12 12 12 20 20 F F F <--- 20 400-419 7 12 12 10 20 20 F F F <--- 21 420-439 11 12 12 19 20 20 F F F <--- 22 440-459 2 12 12 2 20 20 F F F <--- 23 460-479 6 12 12 9 22 22 T F F <--- 24 480-499 6 12 12 9 22 22 F T T <--- 25 500-519 6 6 6 8 8 8 F F F 26 520-539 10 12 12 17 20 20 F F F <--- 27 540-559 11 12 12 19 20 20 F F F <--- 28 560-579 11 11 11 19 19 19 F F F 29 580-599 4 12 12 6 20 20 F F F <--- Max gen: 12, Max flush: 22 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 9 9 9 13 13 13 F F F 2 040-059 7 7 7 10 10 10 F F F 3 060-079 9 9 9 13 13 13 F F F 4 080-099 5 5 5 7 7 7 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 13 13 13 23 23 23 F F F 7 140-159 13 13 13 23 23 23 F F F 8 160-179 12 12 12 21 21 21 F F F 9 180-199 12 12 12 20 20 20 F F F 10 200-219 10 10 10 15 15 15 F F F 11 220-239 10 10 10 18 18 18 F F F 12 240-259 11 11 11 19 19 19 F F F 13 260-279 11 11 11 19 19 19 F F F 14 280-299 8 8 8 11 11 11 F F F 15 300-319 8 8 8 12 12 12 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 4 4 4 6 6 6 F F F 19 380-399 12 12 12 20 20 20 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 12 12 12 20 20 20 F F F 22 440-459 12 12 12 20 20 20 F F F 23 460-479 12 12 12 22 22 22 F F F 24 480-499 13 13 13 23 23 23 F F F 25 500-519 6 6 6 8 8 8 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 11 11 11 19 19 19 F F F 29 580-599 12 12 12 20 20 20 F F F Max gen: 13, Max flush: 23 Begin loop 7 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.146446326Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.170186428Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.170781206Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.170883829Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.170918167Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.170973106Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248} {"msg":"Crucible 157c601b-26d2-4615-85f5-ff9b3cef3dbf has session id: 1d17883f-21d9-41e9-888b-bb0a427de4f1","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.17181375Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"Upstairs opts: Upstairs UUID: 157c601b-26d2-4615-85f5-ff9b3cef3dbf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.171879009Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"Crucible stats registered with UUID: 157c601b-26d2-4615-85f5-ff9b3cef3dbf","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.171911363Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:07.1719397Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.172434018Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.172525229Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.172562082Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.172917355Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.172952495Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248} {"msg":"157c601b-26d2-4615-85f5-ff9b3cef3dbf active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.173116059Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.1731469Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"client":"0","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.173171498Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"client":"1","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.173194634Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"client":"2","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.17321793Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.173242349Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.173347497Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:40467)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.173436744Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:33675)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.173771937Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:43880)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.173883879Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.175383528Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"client":"2","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.175605087Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"client":"0","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.175966031Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"client":"1","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.176456322Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.176513365Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"[0]R flush_numbers[0..12]: [13, 13, 10, 13, 7, 1, 23, 23, 21, 20, 15, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.176847346Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"downstairs","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"[0]R generation[0..12]: [9, 9, 7, 9, 5, 1, 13, 13, 12, 12, 10, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.176947925Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"downstairs","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.176998636Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"downstairs","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"[1]R flush_numbers[0..12]: [13, 13, 10, 13, 7, 1, 23, 23, 21, 20, 15, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177043464Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"downstairs","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"[1]R generation[0..12]: [9, 9, 7, 9, 5, 1, 13, 13, 12, 12, 10, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177086339Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"downstairs","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177133232Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"downstairs","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"[2]R flush_numbers[0..12]: [13, 13, 10, 13, 7, 1, 23, 23, 21, 20, 15, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177176478Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"downstairs","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"[2]R generation[0..12]: [9, 9, 7, 9, 5, 1, 13, 13, 12, 12, 10, 10]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177219543Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"downstairs","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177264943Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"downstairs","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177315653Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"downstairs","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177355823Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"downstairs","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177393447Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"downstairs","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177441422Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"":"downstairs","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177482985Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177519056Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"client":"0","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177558534Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"client":"1","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177598343Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"client":"2","":"downstairs","session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.17763199Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177663121Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"157c601b-26d2-4615-85f5-ff9b3cef3dbf is now active with session: 1d17883f-21d9-41e9-888b-bb0a427de4f1","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-04T01:08:07.177705625Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.17774818Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248,"session_id":"1d17883f-21d9-41e9-888b-bb0a427de4f1"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177785584Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248} {"msg":"Activated sub_volume 157c601b-26d2-4615-85f5-ff9b3cef3dbf","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:07.177818469Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 159 len 12288 data: 2 3 2 03/30 Read block 228 len 8192 04/30 Read block 434 len 32768 05/30 Read block 397 len 4096 06/30 Write block 224 len 16384 data: 2 2 2 2 07/30 Write block 273 len 24576 data: 3 3 2 2 2 2 08/30 Flush 09/30 Write block 300 len 4096 data: 3 10/30 Write block 457 len 24576 data: 4 4 3 3 2 2 11/30 Write block 113 len 32768 data: 2 2 2 2 2 2 2 2 12/30 Flush 13/30 Flush 14/30 Write block 21 len 16384 data: 4 4 3 3 15/30 Read block 577 len 16384 16/30 Write block 59 len 32768 data: 2 2 2 2 2 4 3 2 17/30 Read block 50 len 16384 18/30 Read block 62 len 24576 19/30 Read block 402 len 4096 20/30 Read block 573 len 12288 21/30 Read block 187 len 28672 22/30 Read block 589 len 4096 23/30 Read block 346 len 36864 24/30 Read block 133 len 12288 25/30 Read block 586 len 4096 26/30 Read block 506 len 12288 27/30 Write block 217 len 8192 data: 3 3 28/30 Read block 386 len 8192 29/30 Read block 316 len 8192 30/30 Write block 262 len 8192 data: 2 2 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 3 Done Sent Done false 1002 Acked Read 2 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1000)) Some(JobId(1012)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:07.207740079Z","hostname":"bmat-EVT22200007-0000ead9","pid":1248} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 9 9 9 13 13 13 T F T <--- 2 040-059 7 7 7 10 10 10 T F T <--- 3 060-079 9 9 9 13 13 13 T F T <--- 4 080-099 5 5 5 7 7 7 F F F 5 100-119 14 1 14 26 1 26 F F F <--- 6 120-139 14 13 14 26 23 26 F F F <--- 7 140-159 14 13 14 25 23 25 F F F <--- 8 160-179 14 12 14 25 21 25 F F F <--- 9 180-199 12 12 12 20 20 20 F F F 10 200-219 10 10 10 15 15 15 T F T <--- 11 220-239 14 10 14 25 18 25 F F F <--- 12 240-259 11 11 11 19 19 19 F F F 13 260-279 14 11 14 25 19 25 T F T <--- 14 280-299 8 8 8 11 11 11 F F F 15 300-319 14 8 14 26 12 26 F F F <--- 16 320-339 10 10 10 14 14 14 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 4 4 4 6 6 6 F F F 19 380-399 12 12 12 20 20 20 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 12 12 12 20 20 20 F F F 22 440-459 14 12 14 26 20 26 F F F <--- 23 460-479 14 12 14 26 22 26 F F F <--- 24 480-499 13 13 13 23 23 23 F F F 25 500-519 6 6 6 8 8 8 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 11 11 11 19 19 19 F F F 29 580-599 12 12 12 20 20 20 F F F Max gen: 14, Max flush: 26 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 15 15 15 27 27 27 F F F 2 040-059 15 15 15 27 27 27 F F F 3 060-079 15 15 15 27 27 27 F F F 4 080-099 5 5 5 7 7 7 F F F 5 100-119 14 14 14 26 26 26 F F F 6 120-139 14 14 14 26 26 26 F F F 7 140-159 14 14 14 25 25 25 F F F 8 160-179 14 14 14 25 25 25 F F F 9 180-199 12 12 12 20 20 20 F F F 10 200-219 15 15 15 27 27 27 F F F 11 220-239 14 14 14 25 25 25 F F F 12 240-259 11 11 11 19 19 19 F F F 13 260-279 15 15 15 27 27 27 F F F 14 280-299 8 8 8 11 11 11 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 7 7 7 10 10 10 F F F 18 360-379 4 4 4 6 6 6 F F F 19 380-399 12 12 12 20 20 20 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 12 12 12 20 20 20 F F F 22 440-459 14 14 14 26 26 26 F F F 23 460-479 14 14 14 26 26 26 F F F 24 480-499 13 13 13 23 23 23 F F F 25 500-519 6 6 6 8 8 8 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 11 11 11 19 19 19 F F F 29 580-599 12 12 12 20 20 20 F F F Max gen: 15, Max flush: 27 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.863620862Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.889403188Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.890067603Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.890216678Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.890250365Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.890295054Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256} {"msg":"Crucible 81522d79-7af2-48b1-878c-3f9aa6c9e945 has session id: 9481d4d6-bfeb-4161-95c0-2a7d2d902216","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.89131365Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"Upstairs opts: Upstairs UUID: 81522d79-7af2-48b1-878c-3f9aa6c9e945, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.891381775Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"Crucible stats registered with UUID: 81522d79-7af2-48b1-878c-3f9aa6c9e945","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.891409841Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:21.891432105Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.891799031Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.891860453Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.891885653Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.89220763Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.892240976Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256} {"msg":"81522d79-7af2-48b1-878c-3f9aa6c9e945 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.892409951Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.892435571Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"client":"0","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.892459118Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"client":"1","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.892482154Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"client":"2","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.892594075Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.892622502Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.892646128Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"ds_connection connected from Ok(127.0.0.1:37668)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.892967525Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:21.89306136Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:21.893096179Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"ds_connection connected from Ok(127.0.0.1:45985)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.893123684Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:21.893155898Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.893599195Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.893646389Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.894729744Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"client":"1","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.89487457Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"client":"0","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.895418948Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:21.895894059Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.895096316Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"ds_connection connected from Ok(127.0.0.1:47072)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.89520965Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.897507309Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"client":"2","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"[0]R flush_numbers[0..12]: [13, 27, 27, 27, 7, 26, 26, 25, 25, 20, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898463001Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"downstairs","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"[0]R generation[0..12]: [9, 15, 15, 15, 5, 14, 14, 14, 14, 12, 15, 14]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898630072Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"downstairs","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898685211Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"downstairs","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"[1]R flush_numbers[0..12]: [13, 27, 27, 27, 7, 26, 26, 25, 25, 20, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898713888Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"downstairs","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"[1]R generation[0..12]: [9, 15, 15, 15, 5, 14, 14, 14, 14, 12, 15, 14]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898741132Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"downstairs","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898770771Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"downstairs","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"[2]R flush_numbers[0..12]: [13, 27, 27, 27, 7, 26, 26, 25, 25, 20, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898798125Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"downstairs","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"[2]R generation[0..12]: [9, 15, 15, 15, 5, 14, 14, 14, 14, 12, 15, 14]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898825128Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"downstairs","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898854466Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"downstairs","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898878254Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"downstairs","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898902962Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"downstairs","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898925968Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"downstairs","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898949134Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"":"downstairs","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898973833Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.898995896Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"client":"0","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.89902779Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"client":"1","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.899076506Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"client":"2","":"downstairs","session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.899122567Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.899145954Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"81522d79-7af2-48b1-878c-3f9aa6c9e945 is now active with session: 9481d4d6-bfeb-4161-95c0-2a7d2d902216","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.899172827Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.899195371Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256,"session_id":"9481d4d6-bfeb-4161-95c0-2a7d2d902216"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO :"2026-03-04T01:08:31.899218848Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256} {"msg":"Activated sub_volume 81522d79-7af2-48b1-878c-3f9aa6c9e945","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:31.899256122Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 88 len 8192 data: 3 3 03/30 Write block 537 len 4096 data: 3 04/30 Read block 401 len 16384 05/30 Write block 167 len 4096 data: 3 06/30 Write block 368 len 24576 data: 2 2 2 2 2 2 07/30 Write block 474 len 40960 data: 4 4 4 4 4 4 4 4 4 4 08/30 Read block 214 len 28672 09/30 Write block 536 len 16384 data: 3 4 3 3 10/30 Read block 45 len 4096 11/30 Flush 12/30 Write block 355 len 40960 data: 3 3 3 3 3 3 3 3 3 2 13/30 Write block 187 len 8192 data: 3 3 14/30 Read block 168 len 12288 15/30 Write block 534 len 12288 data: 3 2 4 16/30 Write block 69 len 8192 data: 2 2 17/30 Write block 133 len 28672 data: 2 3 3 4 4 4 4 18/30 Write block 358 len 40960 data: 4 4 4 4 4 4 3 2 2 2 19/30 Read block 582 len 36864 20/30 Write block 259 len 24576 data: 3 3 3 3 3 2 21/30 Read block 468 len 40960 22/30 Read block 456 len 40960 23/30 Read block 355 len 4096 24/30 Write block 175 len 12288 data: 2 4 4 25/30 Write block 120 len 20480 data: 3 2 3 2 2 26/30 Read block 534 len 32768 27/30 Read block 396 len 8192 28/30 Read block 315 len 36864 29/30 Read block 207 len 20480 30/30 Write block 232 len 28672 data: 3 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 2 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Write 6 Done Done Done false 1006 Acked Write 10 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 24 26 Done 28 28 5 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1010)) Some(JobId(1010)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:31.931304012Z","hostname":"bmat-EVT22200007-0000ead9","pid":1256} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 15 15 15 27 27 27 F F F 2 040-059 15 15 15 27 27 27 F F F 3 060-079 15 15 15 27 27 27 T T F <--- 4 080-099 16 16 5 29 29 7 F F T <--- 5 100-119 14 14 14 26 26 26 F F F 6 120-139 14 14 14 26 26 26 T T F <--- 7 140-159 14 14 14 25 25 25 F F F 8 160-179 16 16 14 29 29 25 T T T <--- 9 180-199 12 12 12 20 20 20 T T F <--- 10 200-219 15 15 15 27 27 27 F F F 11 220-239 14 14 14 25 25 25 T T F <--- 12 240-259 11 11 11 19 19 19 T T F <--- 13 260-279 15 15 15 27 27 27 T T F <--- 14 280-299 8 8 8 11 11 11 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 7 7 7 10 10 10 T T F <--- 18 360-379 16 16 4 29 29 6 T T T <--- 19 380-399 12 12 12 20 20 20 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 12 12 12 20 20 20 F F F 22 440-459 14 14 14 26 26 26 F F F 23 460-479 16 16 14 29 29 26 F F F <--- 24 480-499 16 16 13 29 29 23 F F F <--- 25 500-519 6 6 6 8 8 8 F F F 26 520-539 16 16 12 29 29 20 T T T <--- 27 540-559 12 12 12 20 20 20 F F F 28 560-579 11 11 11 19 19 19 F F F 29 580-599 12 12 12 20 20 20 F F F Max gen: 16, Max flush: 29 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 15 15 15 27 27 27 F F F 2 040-059 15 15 15 27 27 27 F F F 3 060-079 17 17 17 30 30 30 F F F 4 080-099 16 16 16 29 29 29 F F F 5 100-119 14 14 14 26 26 26 F F F 6 120-139 17 17 17 30 30 30 F F F 7 140-159 14 14 14 25 25 25 F F F 8 160-179 17 17 17 30 30 30 F F F 9 180-199 17 17 17 30 30 30 F F F 10 200-219 15 15 15 27 27 27 F F F 11 220-239 17 17 17 30 30 30 F F F 12 240-259 17 17 17 30 30 30 F F F 13 260-279 17 17 17 30 30 30 F F F 14 280-299 8 8 8 11 11 11 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 17 17 17 30 30 30 F F F 18 360-379 17 17 17 30 30 30 F F F 19 380-399 12 12 12 20 20 20 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 12 12 12 20 20 20 F F F 22 440-459 14 14 14 26 26 26 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 6 6 6 8 8 8 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 11 11 11 19 19 19 F F F 29 580-599 12 12 12 20 20 20 F F F Max gen: 17, Max flush: 30 Begin loop 9 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.568667689Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:36.592475465Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.592583379Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.615754524Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.616445412Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.616528146Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.616554017Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.616593024Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264} {"msg":"Crucible 87cbeb24-dde0-4ec7-9fab-545d3dfbeacc has session id: ba43aec2-d428-4756-962a-04272589465b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.617710236Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"Upstairs opts: Upstairs UUID: 87cbeb24-dde0-4ec7-9fab-545d3dfbeacc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.617793972Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"Crucible stats registered with UUID: 87cbeb24-dde0-4ec7-9fab-545d3dfbeacc","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.617826757Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:36.617854512Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.618297529Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.6183636Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.618394391Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.618667802Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.618703323Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264} {"msg":"87cbeb24-dde0-4ec7-9fab-545d3dfbeacc active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.618865744Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.618901525Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"client":"0","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.618931975Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"client":"1","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.618961444Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"client":"2","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.618990351Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.619021092Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.619260846Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38544)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.619344332Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"ds_connection connected from Ok(127.0.0.1:65108)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.619374131Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49025)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.619400864Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.621063856Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"client":"1","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.621573205Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"client":"2","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.621628654Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"client":"0","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622028155Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622109526Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"[0]R flush_numbers[0..12]: [13, 27, 27, 30, 29, 26, 30, 25, 30, 30, 27, 30]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622498076Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"downstairs","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"[0]R generation[0..12]: [9, 15, 15, 17, 16, 14, 17, 14, 17, 17, 15, 17]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622538305Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"downstairs","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622568465Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"downstairs","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"[1]R flush_numbers[0..12]: [13, 27, 27, 30, 29, 26, 30, 25, 30, 30, 27, 30]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.62259598Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"downstairs","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"[1]R generation[0..12]: [9, 15, 15, 17, 16, 14, 17, 14, 17, 17, 15, 17]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622623644Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"downstairs","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622653133Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"downstairs","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"[2]R flush_numbers[0..12]: [13, 27, 27, 30, 29, 26, 30, 25, 30, 30, 27, 30]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622680687Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"downstairs","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"[2]R generation[0..12]: [9, 15, 15, 17, 16, 14, 17, 14, 17, 17, 15, 17]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622716638Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"downstairs","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622746568Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"downstairs","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622770615Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"downstairs","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622795404Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"downstairs","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622839852Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"downstairs","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622864922Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"":"downstairs","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622890142Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622912506Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"client":"0","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622937486Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"client":"1","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622962165Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"client":"2","":"downstairs","session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.622987755Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.623036973Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"87cbeb24-dde0-4ec7-9fab-545d3dfbeacc is now active with session: ba43aec2-d428-4756-962a-04272589465b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.623064016Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:08:36.62308619Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264,"session_id":"ba43aec2-d428-4756-962a-04272589465b"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.623127051Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264} {"msg":"Activated sub_volume 87cbeb24-dde0-4ec7-9fab-545d3dfbeacc","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:36.623147762Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 423 len 36864 data: 3 4 4 3 3 3 4 3 3 02/30 Read block 36 len 28672 03/30 Write block 203 len 36864 data: 5 4 4 4 4 4 3 3 3 04/30 Write block 161 len 32768 data: 3 2 3 3 4 3 4 3 05/30 Write block 374 len 4096 data: 2 06/30 Read block 404 len 4096 07/30 Read block 558 len 4096 08/30 Write block 143 len 24576 data: 4 3 3 4 5 5 09/30 Read block 485 len 20480 10/30 Read block 232 len 16384 11/30 Read block 21 len 12288 12/30 Read block 202 len 32768 13/30 Write block 22 len 4096 data: 5 14/30 Write block 234 len 28672 data: 4 4 4 4 4 4 3 15/30 Write block 422 len 36864 data: 3 4 5 5 4 4 4 5 4 16/30 Read block 231 len 24576 17/30 Read block 522 len 40960 18/30 Write block 207 len 20480 data: 5 5 4 4 4 19/30 Read block 47 len 28672 20/30 Write block 49 len 24576 data: 3 3 3 3 3 3 21/30 Write block 99 len 8192 data: 2 2 22/30 Flush 23/30 Write block 490 len 32768 data: 2 2 2 2 3 3 4 4 24/30 Read block 107 len 28672 25/30 Write block 570 len 28672 data: 2 2 2 2 2 2 2 26/30 Write block 86 len 32768 data: 2 3 4 4 3 3 3 3 27/30 Write block 99 len 40960 data: 3 3 2 2 2 2 2 2 2 2 28/30 Write block 45 len 4096 data: 2 29/30 Read block 485 len 36864 30/30 Write block 104 len 8192 data: 3 3 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 7 Sent Done Done false 1002 Acked Write 9 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Read 9 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:36.657323876Z","hostname":"bmat-EVT22200007-0000ead9","pid":1264} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 15 18 18 27 31 31 F F F <--- 2 040-059 15 18 18 27 31 31 F T T <--- 3 060-079 17 17 17 30 30 30 F F F 4 080-099 16 18 18 29 31 31 F T T <--- 5 100-119 14 18 18 26 31 31 F T T <--- 6 120-139 17 17 17 30 30 30 F F F 7 140-159 14 18 18 25 31 31 F F F <--- 8 160-179 17 18 18 30 31 31 F F F <--- 9 180-199 17 17 17 30 30 30 F F F 10 200-219 15 18 18 27 31 31 F F F <--- 11 220-239 17 18 18 30 31 31 F F F <--- 12 240-259 17 18 18 30 31 31 F F F <--- 13 260-279 17 17 17 30 30 30 F F F 14 280-299 8 8 8 11 11 11 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 17 17 17 30 30 30 F F F 18 360-379 17 18 18 30 31 31 F F F <--- 19 380-399 12 12 12 20 20 20 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 12 18 18 20 31 31 T F F <--- 22 440-459 14 14 14 26 26 26 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 16 16 16 29 29 29 F T T <--- 25 500-519 6 6 6 8 8 8 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 11 11 11 19 19 19 F T T <--- 29 580-599 12 12 12 20 20 20 F F F Max gen: 18, Max flush: 31 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 18 18 18 31 31 31 F F F 2 040-059 19 19 19 32 32 32 F F F 3 060-079 17 17 17 30 30 30 F F F 4 080-099 19 19 19 32 32 32 F F F 5 100-119 19 19 19 32 32 32 F F F 6 120-139 17 17 17 30 30 30 F F F 7 140-159 18 18 18 31 31 31 F F F 8 160-179 18 18 18 31 31 31 F F F 9 180-199 17 17 17 30 30 30 F F F 10 200-219 18 18 18 31 31 31 F F F 11 220-239 18 18 18 31 31 31 F F F 12 240-259 18 18 18 31 31 31 F F F 13 260-279 17 17 17 30 30 30 F F F 14 280-299 8 8 8 11 11 11 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 17 17 17 30 30 30 F F F 18 360-379 18 18 18 31 31 31 F F F 19 380-399 12 12 12 20 20 20 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 18 18 18 31 31 31 F F F 22 440-459 14 14 14 26 26 26 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 19 19 19 32 32 32 F F F 25 500-519 6 6 6 8 8 8 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 19 19 19 32 32 32 F F F 29 580-599 12 12 12 20 20 20 F F F Max gen: 19, Max flush: 32 Begin loop 10 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.284945255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.320746346Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.3213847Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.321479187Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.321507984Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.321549627Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272} {"msg":"Crucible 24d015a9-edbe-4101-8ec2-d9210162e370 has session id: 42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.322405361Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"Upstairs opts: Upstairs UUID: 24d015a9-edbe-4101-8ec2-d9210162e370, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.322689794Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"Crucible stats registered with UUID: 24d015a9-edbe-4101-8ec2-d9210162e370","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.322726056Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:41.32275352Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.323199182Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.323254452Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.323285203Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.32347554Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.323511621Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272} {"msg":"24d015a9-edbe-4101-8ec2-d9210162e370 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.323592862Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.323623483Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"client":"0","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.323653251Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"client":"1","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.32368273Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"client":"2","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.323712068Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.323878618Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.323913807Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35965)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.324013174Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47719)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.324131589Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47141)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.324553905Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.326030638Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"client":"2","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.326140776Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"client":"1","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.326357255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.326457884Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.326822707Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"client":"0","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"[0]R flush_numbers[0..12]: [13, 31, 32, 30, 32, 32, 30, 31, 31, 30, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.327593443Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"downstairs","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"[0]R generation[0..12]: [9, 18, 19, 17, 19, 19, 17, 18, 18, 17, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.327628582Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"downstairs","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.327656367Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"downstairs","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"[1]R flush_numbers[0..12]: [13, 31, 32, 30, 32, 32, 30, 31, 31, 30, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.327682148Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"downstairs","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"[1]R generation[0..12]: [9, 18, 19, 17, 19, 19, 17, 18, 18, 17, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.32770822Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"downstairs","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.327735504Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"downstairs","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"[2]R flush_numbers[0..12]: [13, 31, 32, 30, 32, 32, 30, 31, 31, 30, 31, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.327760894Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"downstairs","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"[2]R generation[0..12]: [9, 18, 19, 17, 19, 19, 17, 18, 18, 17, 18, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.327786234Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"downstairs","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.327813578Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"downstairs","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.327836233Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"downstairs","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.327859419Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"downstairs","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.327881593Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"downstairs","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.327904037Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"":"downstairs","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:08:41.327927815Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.327965229Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"client":"0","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.328012973Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"client":"1","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.328039175Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"client":"2","":"downstairs","session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.328065036Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.328088813Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"24d015a9-edbe-4101-8ec2-d9210162e370 is now active with session: 42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.328115737Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.328138171Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272,"session_id":"42ccb7b6-0b49-4d6f-87e8-4ee5a9a6a21e"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.328161547Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272} {"msg":"Activated sub_volume 24d015a9-edbe-4101-8ec2-d9210162e370","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:41.328181928Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 163 len 36864 data: 4 4 5 4 5 4 2 2 2 02/30 Read block 279 len 24576 03/30 Read block 385 len 28672 04/30 Write block 556 len 40960 data: 3 3 3 3 3 3 3 3 3 2 05/30 Write block 481 len 20480 data: 5 5 5 3 3 06/30 Write block 381 len 20480 data: 3 3 2 2 2 07/30 Read block 94 len 36864 08/30 Read block 131 len 40960 09/30 Flush 10/30 Write block 268 len 28672 data: 2 2 2 2 3 4 4 11/30 Write block 113 len 16384 data: 3 3 3 3 12/30 Flush 13/30 Write block 541 len 12288 data: 5 5 4 14/30 Read block 314 len 28672 15/30 Write block 68 len 32768 data: 2 3 3 2 2 2 2 2 16/30 Flush 17/30 Write block 200 len 16384 data: 4 5 5 6 18/30 Read block 74 len 8192 19/30 Flush 20/30 Write block 143 len 16384 data: 5 4 4 5 21/30 Read block 127 len 12288 22/30 Read block 340 len 40960 23/30 Write block 111 len 40960 data: 2 2 4 4 4 4 3 3 3 4 24/30 Read block 306 len 36864 25/30 Write block 545 len 20480 data: 2 2 3 3 3 26/30 Read block 377 len 8192 27/30 Read block 64 len 32768 28/30 Read block 344 len 28672 29/30 Read block 427 len 8192 30/30 Read block 486 len 24576 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Sent Done Done false 1001 Acked Read 6 Sent Done Done false 1002 Acked Read 7 Sent Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:41.365574901Z","hostname":"bmat-EVT22200007-0000ead9","pid":1272} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 18 18 18 31 31 31 F F F 2 040-059 19 19 19 32 32 32 F F F 3 060-079 17 20 20 30 35 35 F F F <--- 4 080-099 19 19 19 32 32 32 F F F 5 100-119 19 20 20 32 34 34 F T T <--- 6 120-139 17 17 17 30 30 30 F T T <--- 7 140-159 18 18 18 31 31 31 F T T <--- 8 160-179 18 20 20 31 33 33 F F F <--- 9 180-199 17 17 17 30 30 30 F F F 10 200-219 18 20 20 31 36 36 F F F <--- 11 220-239 18 18 18 31 31 31 F F F 12 240-259 18 18 18 31 31 31 F F F 13 260-279 17 20 20 30 34 34 F F F <--- 14 280-299 8 8 8 11 11 11 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 17 17 17 30 30 30 F F F 18 360-379 18 18 18 31 31 31 F F F 19 380-399 12 20 20 20 33 33 F F F <--- 20 400-419 12 12 12 20 20 20 F F F 21 420-439 18 18 18 31 31 31 F F F 22 440-459 14 14 14 26 26 26 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 19 20 20 32 33 33 F F F <--- 25 500-519 6 6 6 8 8 8 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 12 20 20 20 35 35 F T T <--- 28 560-579 19 20 20 32 33 33 F F F <--- 29 580-599 12 12 12 20 20 20 F F F Max gen: 20, Max flush: 36 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 18 18 18 31 31 31 F F F 2 040-059 19 19 19 32 32 32 F F F 3 060-079 20 20 20 35 35 35 F F F 4 080-099 19 19 19 32 32 32 F F F 5 100-119 21 21 21 37 37 37 F F F 6 120-139 21 21 21 37 37 37 F F F 7 140-159 21 21 21 37 37 37 F F F 8 160-179 20 20 20 33 33 33 F F F 9 180-199 17 17 17 30 30 30 F F F 10 200-219 20 20 20 36 36 36 F F F 11 220-239 18 18 18 31 31 31 F F F 12 240-259 18 18 18 31 31 31 F F F 13 260-279 20 20 20 34 34 34 F F F 14 280-299 8 8 8 11 11 11 F F F 15 300-319 14 14 14 26 26 26 F F F 16 320-339 10 10 10 14 14 14 F F F 17 340-359 17 17 17 30 30 30 F F F 18 360-379 18 18 18 31 31 31 F F F 19 380-399 20 20 20 33 33 33 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 18 18 18 31 31 31 F F F 22 440-459 14 14 14 26 26 26 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 6 6 6 8 8 8 F F F 26 520-539 17 17 17 30 30 30 F F F 27 540-559 21 21 21 37 37 37 F F F 28 560-579 20 20 20 33 33 33 F F F 29 580-599 12 12 12 20 20 20 F F F Max gen: 21, Max flush: 37 Begin loop 11 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:45.975657174Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.002625448Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.003314091Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.0034144Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.003450561Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.003507805Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280} {"msg":"Crucible 3a34111b-9117-4550-8abd-3316de292d72 has session id: 97137a04-e0c2-47d4-843f-8f4e86b0fab7","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.004591049Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"Upstairs opts: Upstairs UUID: 3a34111b-9117-4550-8abd-3316de292d72, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.004676879Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"Crucible stats registered with UUID: 3a34111b-9117-4550-8abd-3316de292d72","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.004728411Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:46.004769803Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.005589807Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.005687991Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.005718181Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.005746196Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.005769703Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280} {"msg":"3a34111b-9117-4550-8abd-3316de292d72 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.005792087Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.006230766Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"client":"0","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.006554527Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"client":"1","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.006627762Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"client":"2","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.006656559Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.006684895Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.006717429Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"ds_connection connected from Ok(127.0.0.1:54375)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.006747368Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"ds_connection connected from Ok(127.0.0.1:51127)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.006776737Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"ds_connection connected from Ok(127.0.0.1:33927)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.006812537Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.008441362Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"client":"1","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.008491642Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"client":"0","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.008845362Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"client":"2","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.009114776Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.009156528Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"[0]R flush_numbers[0..12]: [13, 31, 32, 35, 32, 37, 37, 37, 33, 30, 36, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.009999247Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"downstairs","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"[0]R generation[0..12]: [9, 18, 19, 20, 19, 21, 21, 21, 20, 17, 20, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.01007685Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"downstairs","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010107381Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"downstairs","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"[1]R flush_numbers[0..12]: [13, 31, 32, 35, 32, 37, 37, 37, 33, 30, 36, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010134064Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"downstairs","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"[1]R generation[0..12]: [9, 18, 19, 20, 19, 21, 21, 21, 20, 17, 20, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010159945Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"downstairs","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010187359Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"downstairs","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"[2]R flush_numbers[0..12]: [13, 31, 32, 35, 32, 37, 37, 37, 33, 30, 36, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010221697Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"downstairs","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"[2]R generation[0..12]: [9, 18, 19, 20, 19, 21, 21, 21, 20, 17, 20, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010247408Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"downstairs","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010284181Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"downstairs","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010310142Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"downstairs","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010333088Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"downstairs","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"Next flush: 38","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:08:46.010355222Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"downstairs","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010388207Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"":"downstairs","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010411994Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010433086Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"client":"0","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010456542Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"client":"1","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010479558Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"client":"2","":"downstairs","session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010503144Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010524988Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"3a34111b-9117-4550-8abd-3316de292d72 is now active with session: 97137a04-e0c2-47d4-843f-8f4e86b0fab7","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010549767Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010570548Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280,"session_id":"97137a04-e0c2-47d4-843f-8f4e86b0fab7"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.010592862Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280} {"msg":"Activated sub_volume 3a34111b-9117-4550-8abd-3316de292d72","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:46.01061223Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 295 len 20480 data: 4 4 4 3 3 02/30 Write block 257 len 32768 data: 3 3 4 4 4 4 4 3 03/30 Read block 85 len 36864 04/30 Read block 167 len 40960 05/30 Read block 62 len 16384 06/30 Write block 532 len 12288 data: 3 3 4 07/30 Read block 62 len 12288 08/30 Write block 563 len 40960 data: 4 4 3 2 2 2 2 3 3 3 09/30 Write block 324 len 16384 data: 3 3 3 3 10/30 Read block 490 len 4096 11/30 Read block 107 len 24576 12/30 Write block 279 len 24576 data: 2 2 2 2 2 2 13/30 Read block 82 len 28672 14/30 Write block 307 len 28672 data: 2 2 2 3 4 3 2 15/30 Read block 310 len 32768 16/30 Read block 471 len 24576 17/30 Read block 130 len 28672 18/30 Read block 274 len 24576 19/30 Read block 51 len 4096 20/30 Read block 471 len 12288 21/30 Write block 76 len 8192 data: 2 2 22/30 Read block 425 len 16384 23/30 Read block 364 len 20480 24/30 Read block 486 len 40960 25/30 Write block 56 len 16384 data: 2 2 2 3 26/30 Read block 522 len 8192 27/30 Write block 285 len 32768 data: 2 2 2 2 2 2 2 2 28/30 Read block 308 len 20480 29/30 Read block 7 len 40960 30/30 Write block 191 len 40960 data: 4 4 4 4 3 4 4 4 4 5 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Sent false 1001 Acked Write 8 Done Done Sent false 1002 Acked Read 9 Done Done Sent false 1003 Acked Read 10 Done Done Sent false 1004 Acked Read 4 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Read 10 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:46.044748255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1280} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 18 18 18 31 31 31 F F F 2 040-059 19 19 19 32 32 32 T T F <--- 3 060-079 20 20 20 35 35 35 T T F <--- 4 080-099 19 19 19 32 32 32 F F F 5 100-119 21 21 21 37 37 37 F F F 6 120-139 21 21 21 37 37 37 F F F 7 140-159 21 21 21 37 37 37 F F F 8 160-179 20 20 20 33 33 33 F F F 9 180-199 17 17 17 30 30 30 T T F <--- 10 200-219 20 20 20 36 36 36 T T F <--- 11 220-239 18 18 18 31 31 31 F F F 12 240-259 18 18 18 31 31 31 T T F <--- 13 260-279 20 20 20 34 34 34 T T F <--- 14 280-299 8 8 8 11 11 11 T T F <--- 15 300-319 14 14 14 26 26 26 T T F <--- 16 320-339 10 10 10 14 14 14 T T F <--- 17 340-359 17 17 17 30 30 30 F F F 18 360-379 18 18 18 31 31 31 F F F 19 380-399 20 20 20 33 33 33 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 18 18 18 31 31 31 F F F 22 440-459 14 14 14 26 26 26 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 6 6 6 8 8 8 F F F 26 520-539 17 17 17 30 30 30 T T F <--- 27 540-559 21 21 21 37 37 37 F F F 28 560-579 20 20 20 33 33 33 T T F <--- 29 580-599 12 12 12 20 20 20 F F F Max gen: 21, Max flush: 37 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 F F F 1 020-039 18 18 18 31 31 31 F F F 2 040-059 22 22 22 38 38 38 F F F 3 060-079 22 22 22 38 38 38 F F F 4 080-099 19 19 19 32 32 32 F F F 5 100-119 21 21 21 37 37 37 F F F 6 120-139 21 21 21 37 37 37 F F F 7 140-159 21 21 21 37 37 37 F F F 8 160-179 20 20 20 33 33 33 F F F 9 180-199 22 22 22 38 38 38 F F F 10 200-219 22 22 22 38 38 38 F F F 11 220-239 18 18 18 31 31 31 F F F 12 240-259 22 22 22 38 38 38 F F F 13 260-279 22 22 22 38 38 38 F F F 14 280-299 22 22 22 38 38 38 F F F 15 300-319 22 22 22 38 38 38 F F F 16 320-339 22 22 22 38 38 38 F F F 17 340-359 17 17 17 30 30 30 F F F 18 360-379 18 18 18 31 31 31 F F F 19 380-399 20 20 20 33 33 33 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 18 18 18 31 31 31 F F F 22 440-459 14 14 14 26 26 26 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 6 6 6 8 8 8 F F F 26 520-539 22 22 22 38 38 38 F F F 27 540-559 21 21 21 37 37 37 F F F 28 560-579 22 22 22 38 38 38 F F F 29 580-599 12 12 12 20 20 20 F F F Max gen: 22, Max flush: 38 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.633560346Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.661652585Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.662164068Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.662248294Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.662281159Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.662325998Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288} {"msg":"Crucible 16c0f57d-d2d5-4b9b-8a3b-de34d1c5b557 has session id: 34262cf4-5a37-4239-b057-155924935ac4","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.663364735Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"Upstairs opts: Upstairs UUID: 16c0f57d-d2d5-4b9b-8a3b-de34d1c5b557, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.663451897Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"Crucible stats registered with UUID: 16c0f57d-d2d5-4b9b-8a3b-de34d1c5b557","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.663487207Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:50.663531826Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.663849334Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.664037076Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.66415046Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.66424626Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.664323864Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288} {"msg":"16c0f57d-d2d5-4b9b-8a3b-de34d1c5b557 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.664398852Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.664482047Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"client":"0","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.664566905Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"client":"1","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.664619349Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"client":"2","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.664671252Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.664725098Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.664778704Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"ds_connection connected from Ok(127.0.0.1:49633)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.664835206Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"ds_connection connected from Ok(127.0.0.1:56109)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.665052937Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"ds_connection connected from Ok(127.0.0.1:59524)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.665121152Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.666530142Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"client":"1","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.666684788Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"client":"0","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.667314585Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.667354314Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"client":"2","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.667496164Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"[0]R flush_numbers[0..12]: [13, 31, 38, 38, 32, 37, 37, 37, 33, 38, 38, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668484151Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"downstairs","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"[0]R generation[0..12]: [9, 18, 22, 22, 19, 21, 21, 21, 20, 22, 22, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.66851925Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"downstairs","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668557255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"downstairs","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"[1]R flush_numbers[0..12]: [13, 31, 38, 38, 32, 37, 37, 37, 33, 38, 38, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668587545Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"downstairs","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"[1]R generation[0..12]: [9, 18, 22, 22, 19, 21, 21, 21, 20, 22, 22, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668617324Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"downstairs","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668649368Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"downstairs","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"[2]R flush_numbers[0..12]: [13, 31, 38, 38, 32, 37, 37, 37, 33, 38, 38, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668678926Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"downstairs","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"[2]R generation[0..12]: [9, 18, 22, 22, 19, 21, 21, 21, 20, 22, 22, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668708585Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"downstairs","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668740177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"downstairs","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"Max found gen is 23","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668766139Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"downstairs","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"Generation requested: 24 >= found:23","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668792982Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"downstairs","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668818422Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"downstairs","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668844073Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"":"downstairs","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668871427Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668895916Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"client":"0","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.6689232Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"client":"1","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668949893Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288Wait for a query_work_queue command to finish before sending IO ,"client":"2","":"downstairs","session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.668998419Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.669024771Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"16c0f57d-d2d5-4b9b-8a3b-de34d1c5b557 is now active with session: 34262cf4-5a37-4239-b057-155924935ac4","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.669053919Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.669078127Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288,"session_id":"34262cf4-5a37-4239-b057-155924935ac4"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.669103777Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288} {"msg":"Activated sub_volume 16c0f57d-d2d5-4b9b-8a3b-de34d1c5b557","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:50.66912556Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 510 len 8192 data: 4 4 02/30 Read block 544 len 40960 03/30 Read block 55 len 32768 04/30 Read block 392 len 32768 05/30 Write block 357 len 24576 data: 4 5 5 5 5 5 06/30 Read block 492 len 36864 07/30 Read block 157 len 32768 08/30 Read block 52 len 16384 09/30 Write block 138 len 12288 data: 5 5 4 10/30 Write block 420 len 28672 data: 3 3 4 5 6 6 5 11/30 Read block 90 len 28672 12/30 Write block 434 len 28672 data: 2 2 2 2 3 3 3 13/30 Write block 183 len 16384 data: 2 2 2 3 14/30 Read block 390 len 36864 15/30 Read block 153 len 24576 16/30 Write block 340 len 20480 data: 2 2 2 2 2 17/30 Read block 218 len 4096 18/30 Write block 291 len 32768 data: 3 3 2 4 5 5 5 4 19/30 Write block 307 len 12288 data: 3 3 3 20/30 Write block 138 len 12288 data: 6 6 5 21/30 Write block 161 len 32768 data: 4 3 5 5 6 5 6 5 22/30 Write block 82 len 24576 data: 2 2 2 2 3 4 23/30 Read block 10 len 32768 24/30 Read block 159 len 16384 25/30 Read block 73 len 24576 26/30 Write block 4 len 16384 data: 3 3 3 3 27/30 Read block 364 len 24576 28/30 Write block 433 len 40960 data: 2 3 3 3 3 4 4 4 3 3 29/30 Read block 53 len 16384 30/30 Read block 250 len 24576 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 8 Done Done Sent false 1003 Acked Read 8 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 28 28 Done 30 30 2 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:50.704716896Z","hostname":"bmat-EVT22200007-0000ead9","pid":1288} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 13 13 13 T T F <--- 1 020-039 18 18 18 31 31 31 F F F 2 040-059 22 22 22 38 38 38 F F F 3 060-079 22 22 22 38 38 38 F F F 4 080-099 19 19 19 32 32 32 T T F <--- 5 100-119 21 21 21 37 37 37 F F F 6 120-139 21 21 21 37 37 37 T T F <--- 7 140-159 21 21 21 37 37 37 T T F <--- 8 160-179 20 20 20 33 33 33 T T F <--- 9 180-199 22 22 22 38 38 38 T T F <--- 10 200-219 22 22 22 38 38 38 F F F 11 220-239 18 18 18 31 31 31 F F F 12 240-259 22 22 22 38 38 38 F F F 13 260-279 22 22 22 38 38 38 F F F 14 280-299 22 22 22 38 38 38 T T F <--- 15 300-319 22 22 22 38 38 38 T T F <--- 16 320-339 22 22 22 38 38 38 F F F 17 340-359 17 17 17 30 30 30 T T F <--- 18 360-379 18 18 18 31 31 31 T T F <--- 19 380-399 20 20 20 33 33 33 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 18 18 18 31 31 31 T T F <--- 22 440-459 14 14 14 26 26 26 T T F <--- 23 460-479 16 16 16 29 29 29 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 6 6 6 8 8 8 T T T 26 520-539 22 22 22 38 38 38 F F F 27 540-559 21 21 21 37 37 37 F F F 28 560-579 22 22 22 38 38 38 F F F 29 580-599 12 12 12 20 20 20 F F F Max gen: 22, Max flush: 38 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 39 39 39 F F F 1 020-039 18 18 18 31 31 31 F F F 2 040-059 22 22 22 38 38 38 F F F 3 060-079 22 22 22 38 38 38 F F F 4 080-099 23 23 23 39 39 39 F F F 5 100-119 21 21 21 37 37 37 F F F 6 120-139 23 23 23 39 39 39 F F F 7 140-159 23 23 23 39 39 39 F F F 8 160-179 23 23 23 39 39 39 F F F 9 180-199 23 23 23 39 39 39 F F F 10 200-219 22 22 22 38 38 38 F F F 11 220-239 18 18 18 31 31 31 F F F 12 240-259 22 22 22 38 38 38 F F F 13 260-279 22 22 22 38 38 38 F F F 14 280-299 23 23 23 39 39 39 F F F 15 300-319 23 23 23 39 39 39 F F F 16 320-339 22 22 22 38 38 38 F F F 17 340-359 23 23 23 39 39 39 F F F 18 360-379 23 23 23 39 39 39 F F F 19 380-399 20 20 20 33 33 33 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 23 23 23 39 39 39 F F F 22 440-459 23 23 23 39 39 39 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 23 23 23 39 39 39 F F F 26 520-539 22 22 22 38 38 38 F F F 27 540-559 21 21 21 37 37 37 F F F 28 560-579 22 22 22 38 38 38 F F F 29 580-599 12 12 12 20 20 20 F F F Max gen: 23, Max flush: 39 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.317332034Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.338136795Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.339090523Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.339157756Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.339179139Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.339206904Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296} {"msg":"Crucible 00b72990-8aa8-4b08-a3e7-243ab13951d0 has session id: 6d662932-1bf7-4571-a1ae-3ef6f26acf08","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.3403162Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"Upstairs opts: Upstairs UUID: 00b72990-8aa8-4b08-a3e7-243ab13951d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.34037184Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"Crucible stats registered with UUID: 00b72990-8aa8-4b08-a3e7-243ab13951d0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.340397661Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:55.340426288Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.340824115Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.340875176Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.340899004Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.341546616Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.341579992Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296} {"msg":"00b72990-8aa8-4b08-a3e7-243ab13951d0 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.341757664Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.341784347Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"client":"0","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.34180614Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"client":"1","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.341828183Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"client":"2","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.341958481Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.341987769Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.342081084Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"ds_connection connected from Ok(127.0.0.1:54826)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.342349836Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"ds_connection connected from Ok(127.0.0.1:57782)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.342387581Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"ds_connection connected from Ok(127.0.0.1:62725)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.342412119Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.34422075Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"client":"0","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.344343714Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"client":"1","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.345066064Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.34545796Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.345839255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"client":"2","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"[0]R flush_numbers[0..12]: [39, 31, 38, 38, 39, 37, 39, 39, 39, 39, 38, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.346702945Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"downstairs","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"[0]R generation[0..12]: [23, 18, 22, 22, 23, 21, 23, 23, 23, 23, 22, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.346776691Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"downstairs","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.346843784Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"downstairs","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"[1]R flush_numbers[0..12]: [39, 31, 38, 38, 39, 37, 39, 39, 39, 39, 38, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.346890736Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"downstairs","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"[1]R generation[0..12]: [23, 18, 22, 22, 23, 21, 23, 23, 23, 23, 22, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.346932659Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"downstairs","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.346977629Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"downstairs","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"[2]R flush_numbers[0..12]: [39, 31, 38, 38, 39, 37, 39, 39, 39, 39, 38, 31]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347019501Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"downstairs","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"[2]R generation[0..12]: [23, 18, 22, 22, 23, 21, 23, 23, 23, 23, 22, 18]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347061575Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"downstairs","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:08:55.347111303Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"downstairs","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347182725Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"downstairs","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"Generation requested: 26 >= found:24","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347234036Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"downstairs","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347270448Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"downstairs","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347306429Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"":"downstairs","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347344365Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347378212Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"client":"0","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-04T01:08:55.347416107Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"client":"1","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347457278Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"client":"2","":"downstairs","session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347497738Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347536916Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"00b72990-8aa8-4b08-a3e7-243ab13951d0 is now active with session: 6d662932-1bf7-4571-a1ae-3ef6f26acf08","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347577677Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347611373Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296,"session_id":"6d662932-1bf7-4571-a1ae-3ef6f26acf08"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347647004Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296} {"msg":"Activated sub_volume 00b72990-8aa8-4b08-a3e7-243ab13951d0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:55.347677384Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 533 len 16384 data: 4 5 3 5 04/30 Write block 552 len 12288 data: 3 2 3 05/30 Read block 492 len 24576 06/30 Flush 07/30 Write block 138 len 24576 data: 7 7 6 5 5 6 08/30 Read block 186 len 12288 09/30 Write block 268 len 16384 data: 3 3 3 3 10/30 Read block 260 len 32768 11/30 Read block 329 len 36864 12/30 Read block 177 len 12288 13/30 Write block 322 len 16384 data: 3 3 4 4 14/30 Read block 16 len 8192 15/30 Read block 408 len 12288 16/30 Write block 324 len 4096 data: 5 17/30 Read block 530 len 24576 18/30 Write block 379 len 4096 data: 2 19/30 Read block 156 len 32768 20/30 Read block 438 len 8192 21/30 Write block 574 len 8192 data: 3 3 22/30 Read block 365 len 40960 23/30 Read block 456 len 32768 24/30 Write block 310 len 8192 data: 4 5 25/30 Read block 425 len 12288 26/30 Flush 27/30 Write block 578 len 36864 data: 2 2 2 2 2 2 2 2 2 28/30 Read block 507 len 16384 29/30 Write block 233 len 32768 data: 4 5 5 5 5 5 5 4 30/30 Read block 54 len 4096 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 4 Done Done Sent false 1003 Acked Write 3 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 28 28 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:08:55.378367747Z","hostname":"bmat-EVT22200007-0000ead9","pid":1296} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 39 39 39 F F F 1 020-039 18 18 18 31 31 31 F F F 2 040-059 22 22 22 38 38 38 F F F 3 060-079 22 22 22 38 38 38 F F F 4 080-099 23 23 23 39 39 39 F F F 5 100-119 21 21 21 37 37 37 F F F 6 120-139 26 26 23 43 43 39 F F F <--- 7 140-159 26 26 23 43 43 39 F F F <--- 8 160-179 23 23 23 39 39 39 F F F 9 180-199 23 23 23 39 39 39 F F F 10 200-219 22 22 22 38 38 38 F F F 11 220-239 18 18 18 31 31 31 T T F <--- 12 240-259 22 22 22 38 38 38 T T F <--- 13 260-279 26 26 22 43 43 38 F F F <--- 14 280-299 23 23 23 39 39 39 F F F 15 300-319 26 26 23 43 43 39 F F F <--- 16 320-339 26 26 22 43 43 38 F F F <--- 17 340-359 23 23 23 39 39 39 F F F 18 360-379 26 26 23 43 43 39 F F F <--- 19 380-399 20 20 20 33 33 33 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 23 23 23 39 39 39 F F F 22 440-459 23 23 23 39 39 39 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 23 23 23 39 39 39 F F F 26 520-539 26 26 22 42 42 38 F F F <--- 27 540-559 26 26 21 42 42 37 F F F <--- 28 560-579 26 26 22 43 43 38 T T F <--- 29 580-599 12 12 12 20 20 20 T T F <--- Max gen: 26, Max flush: 43 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 39 39 39 F F F 1 020-039 18 18 18 31 31 31 F F F 2 040-059 22 22 22 38 38 38 F F F 3 060-079 22 22 22 38 38 38 F F F 4 080-099 23 23 23 39 39 39 F F F 5 100-119 21 21 21 37 37 37 F F F 6 120-139 26 26 26 43 43 43 F F F 7 140-159 26 26 26 43 43 43 F F F 8 160-179 23 23 23 39 39 39 F F F 9 180-199 23 23 23 39 39 39 F F F 10 200-219 22 22 22 38 38 38 F F F 11 220-239 27 27 27 44 44 44 F F F 12 240-259 27 27 27 44 44 44 F F F 13 260-279 26 26 26 43 43 43 F F F 14 280-299 23 23 23 39 39 39 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 26 26 26 43 43 43 F F F 17 340-359 23 23 23 39 39 39 F F F 18 360-379 26 26 26 43 43 43 F F F 19 380-399 20 20 20 33 33 33 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 23 23 23 39 39 39 F F F 22 440-459 23 23 23 39 39 39 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 23 23 23 39 39 39 F F F 26 520-539 26 26 26 42 42 42 F F F 27 540-559 26 26 26 42 42 42 F F F 28 560-579 27 27 27 44 44 44 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 27, Max flush: 44 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:08:59.971904873Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.012070857Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.012716345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.01278465Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.012815661Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.012848897Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304} {"msg":"Crucible 613159ba-e979-4a34-a2e8-64059301ef0b has session id: 7dbac3d6-8389-4cd6-a4a0-848ddd6250b5","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.014074153Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"Upstairs opts: Upstairs UUID: 613159ba-e979-4a34-a2e8-64059301ef0b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.014131336Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"Crucible stats registered with UUID: 613159ba-e979-4a34-a2e8-64059301ef0b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.014158871Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:00.014182698Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.014632849Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.014767967Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.014798838Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.014992381Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.015020376Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304} {"msg":"613159ba-e979-4a34-a2e8-64059301ef0b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.015246304Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.015273397Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"client":"0","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.015298687Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"client":"1","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.015323286Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"client":"2","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.015374708Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.015409617Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.015529544Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46236)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.015756664Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:60122)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.015832234Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35147)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.015927132Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.017605535Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"client":"1","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.018089833Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"client":"2","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.018127618Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"client":"0","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.01840657Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.018757636Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"[0]R flush_numbers[0..12]: [39, 31, 38, 38, 39, 37, 43, 43, 39, 39, 38, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.018923564Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"downstairs","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"[0]R generation[0..12]: [23, 18, 22, 22, 23, 21, 26, 26, 23, 23, 22, 27]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.01896199Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"downstairs","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.018992631Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"downstairs","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"[1]R flush_numbers[0..12]: [39, 31, 38, 38, 39, 37, 43, 43, 39, 39, 38, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019020777Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"downstairs","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"[1]R generation[0..12]: [23, 18, 22, 22, 23, 21, 26, 26, 23, 23, 22, 27]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019056537Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"downstairs","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019086096Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"downstairs","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"[2]R flush_numbers[0..12]: [39, 31, 38, 38, 39, 37, 43, 43, 39, 39, 38, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.01911359Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"downstairs","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"[2]R generation[0..12]: [23, 18, 22, 22, 23, 21, 26, 26, 23, 23, 22, 27]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019140794Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"downstairs","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019170363Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"downstairs","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019194701Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"downstairs","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:09:00.019316682Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"downstairs","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019536267Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"downstairs","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019582138Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"":"downstairs","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019607428Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019629061Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"client":"0","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019652818Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"client":"1","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019675984Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"client":"2","":"downstairs","session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.01969949Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019721684Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"613159ba-e979-4a34-a2e8-64059301ef0b is now active with session: 7dbac3d6-8389-4cd6-a4a0-848ddd6250b5","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019746654Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019773807Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304,"session_id":"7dbac3d6-8389-4cd6-a4a0-848ddd6250b5"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019796092Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304} {"msg":"Activated sub_volume 613159ba-e979-4a34-a2e8-64059301ef0b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:00.019814989Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 168 len 8192 data: 6 3 02/30 Read block 374 len 28672 03/30 Read block 468 len 24576 04/30 Write block 555 len 40960 data: 3 4 4 4 4 4 4 4 5 5 05/30 Read block 139 len 4096 06/30 Write block 68 len 12288 data: 3 4 4 07/30 Read block 386 len 20480 08/30 Write block 4 len 24576 data: 4 4 4 4 3 3 09/30 Read block 139 len 28672 10/30 Read block 572 len 20480 11/30 Flush 12/30 Flush 13/30 Write block 391 len 8192 data: 3 4 14/30 Read block 515 len 16384 15/30 Write block 304 len 40960 data: 2 2 2 4 4 4 5 6 4 3 16/30 Write block 458 len 40960 data: 5 4 4 3 3 2 2 2 2 2 17/30 Write block 210 len 16384 data: 5 5 3 2 18/30 Write block 512 len 16384 data: 4 4 4 3 19/30 Write block 446 len 36864 data: 3 2 2 2 2 3 3 3 4 20/30 Write block 514 len 24576 data: 5 4 3 3 3 3 21/30 Write block 291 len 32768 data: 4 4 3 5 6 6 6 5 22/30 Read block 455 len 24576 23/30 Read block 515 len 40960 24/30 Write block 433 len 24576 data: 3 4 4 4 4 5 25/30 Read block 49 len 20480 26/30 Flush 27/30 Write block 341 len 8192 data: 3 3 28/30 Write block 562 len 32768 data: 5 6 6 4 3 3 3 3 29/30 Write block 322 len 4096 data: 4 30/30 Write block 249 len 8192 data: 5 5 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 2 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Write 9 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 3 3 32 Done 4 27 27 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:00.05271671Z","hostname":"bmat-EVT22200007-0000ead9","pid":1304} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 28 28 39 45 45 F F F <--- 1 020-039 18 18 18 31 31 31 F F F 2 040-059 22 22 22 38 38 38 F F F 3 060-079 22 28 28 38 45 45 F F F <--- 4 080-099 23 23 23 39 39 39 F F F 5 100-119 21 21 21 37 37 37 F F F 6 120-139 26 26 26 43 43 43 F F F 7 140-159 26 26 26 43 43 43 F F F 8 160-179 23 28 28 39 45 45 T F F <--- 9 180-199 23 23 23 39 39 39 F F F 10 200-219 22 28 28 38 47 47 F F F <--- 11 220-239 27 27 27 44 44 44 F F F 12 240-259 27 27 27 44 44 44 F T T <--- 13 260-279 26 26 26 43 43 43 F F F 14 280-299 23 28 28 39 47 47 F F F <--- 15 300-319 26 28 28 43 47 47 F F F <--- 16 320-339 26 26 26 43 43 43 F T T <--- 17 340-359 23 23 23 39 39 39 F T T <--- 18 360-379 26 26 26 43 43 43 F F F 19 380-399 20 28 28 33 47 47 F F F <--- 20 400-419 12 12 12 20 20 20 F F F 21 420-439 23 28 28 39 47 47 F F F <--- 22 440-459 23 28 28 39 47 47 F F F <--- 23 460-479 16 28 28 29 47 47 F F F <--- 24 480-499 20 20 20 33 33 33 F F F 25 500-519 23 28 28 39 47 47 F F F <--- 26 520-539 26 26 26 42 42 42 F F F 27 540-559 26 28 28 42 45 45 T F F <--- 28 560-579 27 28 28 44 45 45 T T T <--- 29 580-599 27 27 27 44 44 44 F F F Max gen: 28, Max flush: 47 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 F F F 1 020-039 18 18 18 31 31 31 F F F 2 040-059 22 22 22 38 38 38 F F F 3 060-079 28 28 28 45 45 45 F F F 4 080-099 23 23 23 39 39 39 F F F 5 100-119 21 21 21 37 37 37 F F F 6 120-139 26 26 26 43 43 43 F F F 7 140-159 26 26 26 43 43 43 F F F 8 160-179 28 28 28 45 45 45 F F F 9 180-199 23 23 23 39 39 39 F F F 10 200-219 28 28 28 47 47 47 F F F 11 220-239 27 27 27 44 44 44 F F F 12 240-259 29 29 29 48 48 48 F F F 13 260-279 26 26 26 43 43 43 F F F 14 280-299 28 28 28 47 47 47 F F F 15 300-319 28 28 28 47 47 47 F F F 16 320-339 29 29 29 48 48 48 F F F 17 340-359 29 29 29 48 48 48 F F F 18 360-379 26 26 26 43 43 43 F F F 19 380-399 28 28 28 47 47 47 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 28 28 28 47 47 47 F F F 22 440-459 28 28 28 47 47 47 F F F 23 460-479 28 28 28 47 47 47 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 28 28 28 47 47 47 F F F 26 520-539 26 26 26 42 42 42 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 29 29 29 48 48 48 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 29, Max flush: 48 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.746582022Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.777526147Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.778093791Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.77819423Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.77822928Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.778276383Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312} {"msg":"Crucible cf328184-69bc-4e02-9e29-dedeb9045e17 has session id: 243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.779036788Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"Upstairs opts: Upstairs UUID: cf328184-69bc-4e02-9e29-dedeb9045e17, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.779120404Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"Crucible stats registered with UUID: cf328184-69bc-4e02-9e29-dedeb9045e17","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.779156535Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:04.77918926Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.779625664Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.779722326Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.779772735Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.779945337Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.779994955Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312} {"msg":"cf328184-69bc-4e02-9e29-dedeb9045e17 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.780139251Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.78022953Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"client":"0","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.780277545Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"client":"1","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.780321011Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"client":"2","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.780363114Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.780521237Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.780572228Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:56326)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.780616025Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:49775)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.780687907Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:36341)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.781008252Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.782889166Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"client":"2","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.783043221Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"client":"0","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.783240792Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"client":"1","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.783716664Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.783766813Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"[0]R flush_numbers[0..12]: [45, 31, 38, 45, 39, 37, 43, 43, 45, 39, 47, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784147938Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"downstairs","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"[0]R generation[0..12]: [28, 18, 22, 28, 23, 21, 26, 26, 28, 23, 28, 27]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784205762Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"downstairs","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784252154Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"downstairs","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"[1]R flush_numbers[0..12]: [45, 31, 38, 45, 39, 37, 43, 43, 45, 39, 47, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784293917Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"downstairs","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"[1]R generation[0..12]: [28, 18, 22, 28, 23, 21, 26, 26, 28, 23, 28, 27]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784334978Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"downstairs","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784379196Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"downstairs","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"[2]R flush_numbers[0..12]: [45, 31, 38, 45, 39, 37, 43, 43, 45, 39, 47, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784420708Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"downstairs","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"[2]R generation[0..12]: [28, 18, 22, 28, 23, 21, 26, 26, 28, 23, 28, 27]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784461479Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784516838Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"downstairs","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784552479Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"downstairs","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784589903Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"downstairs","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784625072Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"downstairs","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784666224Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"":"downstairs","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784704119Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784737946Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"client":"0","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.78477535Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"client":"1","":"downstairs","session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784812564Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"client":"2","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784859316Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784895227Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"cf328184-69bc-4e02-9e29-dedeb9045e17 is now active with session: 243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784935156Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.784968593Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312,"session_id":"243bfc6e-7f0f-4c74-b973-5ed5dd22a1e0"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.785003872Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312} {"msg":"Activated sub_volume cf328184-69bc-4e02-9e29-dedeb9045e17","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:04.785034363Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 457 len 32768 data: 5 6 5 5 4 4 3 3 02/30 Flush 03/30 Write block 382 len 32768 data: 4 3 3 3 2 2 2 2 04/30 Flush 05/30 Write block 131 len 28672 data: 2 2 3 4 4 5 5 06/30 Write block 32 len 4096 data: 2 07/30 Read block 545 len 20480 08/30 Read block 121 len 16384 09/30 Read block 570 len 40960 10/30 Read block 171 len 32768 11/30 Write block 469 len 8192 data: 3 3 12/30 Flush 13/30 Write block 113 len 8192 data: 5 5 14/30 Read block 16 len 16384 15/30 Flush 16/30 Write block 498 len 36864 data: 4 5 3 3 3 3 3 3 3 17/30 Write block 336 len 8192 data: 2 2 18/30 Read block 137 len 4096 19/30 Read block 450 len 8192 20/30 Read block 399 len 20480 21/30 Read block 546 len 4096 22/30 Write block 454 len 40960 data: 5 4 4 6 7 6 6 5 5 4 23/30 Flush 24/30 Write block 180 len 40960 data: 3 3 3 3 3 3 4 4 4 3 25/30 Read block 121 len 40960 26/30 Write block 15 len 4096 data: 4 27/30 Write block 288 len 32768 data: 3 3 3 5 5 4 6 7 28/30 Write block 486 len 8192 data: 3 3 29/30 Read block 114 len 32768 30/30 Write block 162 len 40960 data: 4 6 6 7 6 7 7 4 3 3 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Sent Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 8 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:04.819587323Z","hostname":"bmat-EVT22200007-0000ead9","pid":1312} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 28 28 28 45 45 45 T F T <--- 1 020-039 30 18 30 51 31 51 F F F <--- 2 040-059 22 22 22 38 38 38 F F F 3 060-079 28 28 28 45 45 45 F F F 4 080-099 23 23 23 39 39 39 F F F 5 100-119 30 21 30 52 37 52 F F F <--- 6 120-139 30 26 30 51 43 51 F F F <--- 7 140-159 26 26 26 43 43 43 F F F 8 160-179 28 28 28 45 45 45 T F T <--- 9 180-199 23 23 23 39 39 39 T F T <--- 10 200-219 28 28 28 47 47 47 F F F 11 220-239 27 27 27 44 44 44 F F F 12 240-259 29 29 29 48 48 48 F F F 13 260-279 26 26 26 43 43 43 F F F 14 280-299 28 28 28 47 47 47 T F T <--- 15 300-319 28 28 28 47 47 47 F F F 16 320-339 30 29 30 53 48 53 F F F <--- 17 340-359 29 29 29 48 48 48 F F F 18 360-379 26 26 26 43 43 43 F F F 19 380-399 30 28 30 50 47 50 F F F <--- 20 400-419 12 12 12 20 20 20 F F F 21 420-439 28 28 28 47 47 47 F F F 22 440-459 30 28 30 53 47 53 F F F <--- 23 460-479 30 28 30 53 47 53 F F F <--- 24 480-499 30 20 30 53 33 53 T F T <--- 25 500-519 30 28 30 53 47 53 F F F <--- 26 520-539 26 26 26 42 42 42 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 29 29 29 48 48 48 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 30, Max flush: 53 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 54 54 54 F F F 1 020-039 30 30 30 51 51 51 F F F 2 040-059 22 22 22 38 38 38 F F F 3 060-079 28 28 28 45 45 45 F F F 4 080-099 23 23 23 39 39 39 F F F 5 100-119 30 30 30 52 52 52 F F F 6 120-139 30 30 30 51 51 51 F F F 7 140-159 26 26 26 43 43 43 F F F 8 160-179 31 31 31 54 54 54 F F F 9 180-199 31 31 31 54 54 54 F F F 10 200-219 28 28 28 47 47 47 F F F 11 220-239 27 27 27 44 44 44 F F F 12 240-259 29 29 29 48 48 48 F F F 13 260-279 26 26 26 43 43 43 F F F 14 280-299 31 31 31 54 54 54 F F F 15 300-319 28 28 28 47 47 47 F F F 16 320-339 30 30 30 53 53 53 F F F 17 340-359 29 29 29 48 48 48 F F F 18 360-379 26 26 26 43 43 43 F F F 19 380-399 30 30 30 50 50 50 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 28 28 28 47 47 47 F F F 22 440-459 30 30 30 53 53 53 F F F 23 460-479 30 30 30 53 53 53 F F F 24 480-499 31 31 31 54 54 54 F F F 25 500-519 30 30 30 53 53 53 F F F 26 520-539 26 26 26 42 42 42 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 29 29 29 48 48 48 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 31, Max flush: 54 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.426769124Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.455007743Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.455567571Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.455645175Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.455669413Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.455700013Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320} {"msg":"Crucible f8aa9d79-16cf-44bf-b961-858676c1c21b has session id: 81807be8-04ef-4245-bb93-e209127a907d","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.456611588Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"Upstairs opts: Upstairs UUID: f8aa9d79-16cf-44bf-b961-858676c1c21b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.456666477Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"Crucible stats registered with UUID: f8aa9d79-16cf-44bf-b961-858676c1c21b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.456692048Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:09.45671368Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.457342676Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.457397134Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.457425389Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.457869749Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.457905871Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320} {"msg":"f8aa9d79-16cf-44bf-b961-858676c1c21b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.458108692Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.458136788Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"client":"0","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.458163711Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"client":"1","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.458190134Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"client":"2","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.458216335Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.458375941Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.458413255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48452)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.458442623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58766)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.458738389Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"ds_connection connected from Ok(127.0.0.1:37330)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.458775172Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.461182338Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"client":"0","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.461276514Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"client":"2","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.461316223Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"client":"1","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.461924868Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462032992Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"[0]R flush_numbers[0..12]: [54, 51, 38, 45, 39, 52, 51, 43, 54, 54, 47, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462277646Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"downstairs","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"[0]R generation[0..12]: [31, 30, 22, 28, 23, 30, 30, 26, 31, 31, 28, 27]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462321213Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"downstairs","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462359649Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"downstairs","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"[1]R flush_numbers[0..12]: [54, 51, 38, 45, 39, 52, 51, 43, 54, 54, 47, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462394888Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"downstairs","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"[1]R generation[0..12]: [31, 30, 22, 28, 23, 30, 30, 26, 31, 31, 28, 27]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462430188Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"downstairs","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.46246649Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"downstairs","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"[2]R flush_numbers[0..12]: [54, 51, 38, 45, 39, 52, 51, 43, 54, 54, 47, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462510527Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"downstairs","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"[2]R generation[0..12]: [31, 30, 22, 28, 23, 30, 30, 26, 31, 31, 28, 27]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462546138Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"downstairs","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462583492Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"downstairs","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462614102Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"downstairs","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462645505Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"downstairs","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462675995Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"downstairs","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462706616Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"":"downstairs","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {Wait for a query_work_queue command to finish before sending IO "msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462749831Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462780352Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"client":"0","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462811403Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"client":"1","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462843256Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"client":"2","":"downstairs","session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.4628757Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462905569Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"f8aa9d79-16cf-44bf-b961-858676c1c21b is now active with session: 81807be8-04ef-4245-bb93-e209127a907d","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462940308Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.462968694Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320,"session_id":"81807be8-04ef-4245-bb93-e209127a907d"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.463004135Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320} {"msg":"Activated sub_volume f8aa9d79-16cf-44bf-b961-858676c1c21b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:09.463030597Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 97 len 16384 data: 2 2 4 4 02/30 Read block 351 len 16384 03/30 Write block 403 len 12288 data: 2 2 2 04/30 Read block 278 len 32768 05/30 Flush 06/30 Write block 436 len 16384 data: 5 5 6 5 07/30 Read block 361 len 4096 08/30 Write block 149 len 24576 data: 4 4 4 3 3 4 09/30 Read block 518 len 8192 10/30 Write block 291 len 8192 data: 6 6 11/30 Write block 462 len 20480 data: 6 5 4 3 3 12/30 Write block 444 len 4096 data: 3 13/30 Read block 503 len 36864 14/30 Read block 521 len 16384 15/30 Write block 264 len 36864 data: 4 2 2 2 4 4 4 4 4 16/30 Write block 188 len 16384 data: 5 4 3 5 17/30 Write block 181 len 16384 data: 4 4 4 4 18/30 Write block 45 len 12288 data: 3 2 3 19/30 Read block 543 len 8192 20/30 Write block 226 len 20480 data: 3 3 2 2 3 21/30 Flush 22/30 Write block 249 len 8192 data: 6 6 23/30 Read block 196 len 28672 24/30 Read block 405 len 4096 25/30 Read block 261 len 16384 26/30 Write block 358 len 28672 data: 6 6 6 6 6 5 4 27/30 Write block 47 len 20480 data: 4 3 4 4 4 28/30 Write block 81 len 8192 data: 2 3 29/30 Read block 301 len 4096 30/30 Write block 437 len 8192 data: 6 7 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 4 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Write 7 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 25 1 27 Done 24 0 24 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1004)) Some(JobId(1020)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:09.490652184Z","hostname":"bmat-EVT22200007-0000ead9","pid":1320} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 54 54 54 F F F 1 020-039 30 30 30 51 51 51 F F F 2 040-059 32 22 32 56 38 56 T F T <--- 3 060-079 28 28 28 45 45 45 F F F 4 080-099 32 32 32 55 55 55 T F T <--- 5 100-119 32 32 32 55 55 55 F F F 6 120-139 30 30 30 51 51 51 F F F 7 140-159 32 26 32 56 43 56 F F F <--- 8 160-179 31 31 31 54 54 54 F F F 9 180-199 32 31 32 56 54 56 F F F <--- 10 200-219 28 28 28 47 47 47 F F F 11 220-239 32 27 32 56 44 56 F F F <--- 12 240-259 29 29 29 48 48 48 T F T <--- 13 260-279 32 26 32 56 43 56 F F F <--- 14 280-299 32 31 32 56 54 56 F F F <--- 15 300-319 28 28 28 47 47 47 F F F 16 320-339 30 30 30 53 53 53 F F F 17 340-359 29 29 29 48 48 48 T F T <--- 18 360-379 26 26 26 43 43 43 T F T <--- 19 380-399 30 30 30 50 50 50 F F F 20 400-419 32 32 32 55 55 55 F F F 21 420-439 32 28 32 56 47 56 T F T <--- 22 440-459 32 30 32 56 53 56 F F F <--- 23 460-479 32 30 32 56 53 56 F F F <--- 24 480-499 31 31 31 54 54 54 F F F 25 500-519 30 30 30 53 53 53 F F F 26 520-539 26 26 26 42 42 42 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 29 29 29 48 48 48 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 32, Max flush: 56 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 54 54 54 F F F 1 020-039 30 30 30 51 51 51 F F F 2 040-059 33 33 33 57 57 57 F F F 3 060-079 28 28 28 45 45 45 F F F 4 080-099 33 33 33 57 57 57 F F F 5 100-119 32 32 32 55 55 55 F F F 6 120-139 30 30 30 51 51 51 F F F 7 140-159 32 32 32 56 56 56 F F F 8 160-179 31 31 31 54 54 54 F F F 9 180-199 32 32 32 56 56 56 F F F 10 200-219 28 28 28 47 47 47 F F F 11 220-239 32 32 32 56 56 56 F F F 12 240-259 33 33 33 57 57 57 F F F 13 260-279 32 32 32 56 56 56 F F F 14 280-299 32 32 32 56 56 56 F F F 15 300-319 28 28 28 47 47 47 F F F 16 320-339 30 30 30 53 53 53 F F F 17 340-359 33 33 33 57 57 57 F F F 18 360-379 33 33 33 57 57 57 F F F 19 380-399 30 30 30 50 50 50 F F F 20 400-419 32 32 32 55 55 55 F F F 21 420-439 33 33 33 57 57 57 F F F 22 440-459 32 32 32 56 56 56 F F F 23 460-479 32 32 32 56 56 56 F F F 24 480-499 31 31 31 54 54 54 F F F 25 500-519 30 30 30 53 53 53 F F F 26 520-539 26 26 26 42 42 42 F F F 27 540-559 28 28 28 45 45 45 F F F 28 560-579 29 29 29 48 48 48 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 33, Max flush: 57 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.072830841Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.103431215Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.10394856Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.10405481Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.104092194Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.10415064Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328} {"msg":"Crucible 672ed12d-b515-4eb0-834a-133344af6b45 has session id: f7bc2818-2405-4001-afd4-ad09d22980d9","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.105139517Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"Upstairs opts: Upstairs UUID: 672ed12d-b515-4eb0-834a-133344af6b45, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.105225117Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"Crucible stats registered with UUID: 672ed12d-b515-4eb0-834a-133344af6b45","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.105271018Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:14.105311849Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.105594498Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.105667823Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.105713414Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.106168374Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.106216279Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328} {"msg":"672ed12d-b515-4eb0-834a-133344af6b45 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.106401977Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.106444912Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"client":"0","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.106488759Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"client":"1","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.106531984Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"client":"2","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.106574769Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.106619438Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.106663705Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:14.106720738Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:14.106776989Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:14.106833582Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.106871386Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.106925373Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:56531)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.106988338Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61605)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.107036092Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.108965893Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"client":"1","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.109062655Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"client":"0","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.10970152Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:14.109813932Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.108257344Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:53145)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.108494333Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.110719098Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"client":"2","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"[0]R flush_numbers[0..12]: [54, 51, 57, 45, 57, 55, 51, 56, 54, 56, 47, 56]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.11174047Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"downstairs","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"[0]R generation[0..12]: [31, 30, 33, 28, 33, 32, 30, 32, 31, 32, 28, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.111795519Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"downstairs","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.111833484Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"downstairs","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"[1]R flush_numbers[0..12]: [54, 51, 57, 45, 57, 55, 51, 56, 54, 56, 47, 56]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.111867281Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"downstairs","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"[1]R generation[0..12]: [31, 30, 33, 28, 33, 32, 30, 32, 31, 32, 28, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.111899946Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"downstairs","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.111935186Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"downstairs","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"[2]R flush_numbers[0..12]: [54, 51, 57, 45, 57, 55, 51, 56, 54, 56, 47, 56]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.111968051Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"downstairs","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"[2]R generation[0..12]: [31, 30, 33, 28, 33, 32, 30, 32, 31, 32, 28, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112036907Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"downstairs","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112081676Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"downstairs","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112111044Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"downstairs","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112141063Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"downstairs","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.11216975Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"downstairs","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112198156Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"":"downstairs","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112227965Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112254788Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"client":"0","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112284597Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"client":"1","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112314486Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"client":"2","":"downstairs","session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:09:24.112355668Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112403883Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"672ed12d-b515-4eb0-834a-133344af6b45 is now active with session: f7bc2818-2405-4001-afd4-ad09d22980d9","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112440356Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112467149Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328,"session_id":"f7bc2818-2405-4001-afd4-ad09d22980d9"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112495465Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328} {"msg":"Activated sub_volume 672ed12d-b515-4eb0-834a-133344af6b45","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:24.112519843Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 513 len 24576 data: 5 6 5 4 4 4 02/30 Write block 96 len 16384 data: 2 3 3 5 03/30 Read block 408 len 40960 04/30 Read block 17 len 4096 05/30 Read block 170 len 24576 06/30 Write block 538 len 32768 data: 4 4 4 6 6 5 3 3 07/30 Write block 406 len 4096 data: 2 08/30 Read block 570 len 32768 09/30 Read block 52 len 20480 10/30 Write block 231 len 12288 data: 3 4 5 11/30 Write block 279 len 24576 data: 3 3 3 3 3 3 12/30 Write block 502 len 12288 data: 4 4 4 13/30 Read block 282 len 24576 14/30 Write block 522 len 40960 data: 3 3 3 3 3 2 2 3 3 3 15/30 Read block 518 len 20480 16/30 Read block 257 len 8192 17/30 Read block 383 len 36864 18/30 Read block 209 len 24576 19/30 Write block 113 len 24576 data: 6 6 5 5 4 4 20/30 Write block 144 len 28672 data: 5 5 6 6 6 5 5 21/30 Write block 373 len 4096 data: 3 22/30 Read block 190 len 24576 23/30 Read block 387 len 16384 24/30 Read block 253 len 36864 25/30 Read block 300 len 8192 26/30 Read block 432 len 40960 27/30 Read block 425 len 12288 28/30 Read block 63 len 12288 29/30 Read block 223 len 8192 30/30 Write block 437 len 40960 data: 7 8 6 5 4 4 3 4 3 4 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Write 8 Done Done Sent false 1006 Acked Write 1 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Write 3 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Read 10 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 25 27 Done 29 29 5 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:24.145749694Z","hostname":"bmat-EVT22200007-0000ead9","pid":1328} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 54 54 54 F F F 1 020-039 30 30 30 51 51 51 F F F 2 040-059 33 33 33 57 57 57 F F F 3 060-079 28 28 28 45 45 45 F F F 4 080-099 33 33 33 57 57 57 T T T 5 100-119 32 32 32 55 55 55 T T F <--- 6 120-139 30 30 30 51 51 51 F F F 7 140-159 32 32 32 56 56 56 T T F <--- 8 160-179 31 31 31 54 54 54 F F F 9 180-199 32 32 32 56 56 56 F F F 10 200-219 28 28 28 47 47 47 F F F 11 220-239 32 32 32 56 56 56 T T F <--- 12 240-259 33 33 33 57 57 57 F F F 13 260-279 32 32 32 56 56 56 T T F <--- 14 280-299 32 32 32 56 56 56 T T F <--- 15 300-319 28 28 28 47 47 47 F F F 16 320-339 30 30 30 53 53 53 F F F 17 340-359 33 33 33 57 57 57 F F F 18 360-379 33 33 33 57 57 57 T T F <--- 19 380-399 30 30 30 50 50 50 F F F 20 400-419 32 32 32 55 55 55 T T F <--- 21 420-439 33 33 33 57 57 57 T T F <--- 22 440-459 32 32 32 56 56 56 T T F <--- 23 460-479 32 32 32 56 56 56 F F F 24 480-499 31 31 31 54 54 54 F F F 25 500-519 30 30 30 53 53 53 T T T 26 520-539 26 26 26 42 42 42 T T F <--- 27 540-559 28 28 28 45 45 45 T T F <--- 28 560-579 29 29 29 48 48 48 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 33, Max flush: 57 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 54 54 54 F F F 1 020-039 30 30 30 51 51 51 F F F 2 040-059 33 33 33 57 57 57 F F F 3 060-079 28 28 28 45 45 45 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 30 30 30 51 51 51 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 31 31 31 54 54 54 F F F 9 180-199 32 32 32 56 56 56 F F F 10 200-219 28 28 28 47 47 47 F F F 11 220-239 34 34 34 58 58 58 F F F 12 240-259 33 33 33 57 57 57 F F F 13 260-279 34 34 34 58 58 58 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 28 28 28 47 47 47 F F F 16 320-339 30 30 30 53 53 53 F F F 17 340-359 33 33 33 57 57 57 F F F 18 360-379 34 34 34 58 58 58 F F F 19 380-399 30 30 30 50 50 50 F F F 20 400-419 34 34 34 58 58 58 F F F 21 420-439 34 34 34 58 58 58 F F F 22 440-459 34 34 34 58 58 58 F F F 23 460-479 32 32 32 56 56 56 F F F 24 480-499 31 31 31 54 54 54 F F F 25 500-519 34 34 34 58 58 58 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 34 34 34 58 58 58 F F F 28 560-579 29 29 29 48 48 48 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 34, Max flush: 58 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.783958203Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.809252392Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.810013389Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.810184338Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.810214367Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.810249857Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336} {"msg":"Crucible b6093006-403d-44fd-8ca9-53b264952a03 has session id: 49372878-50c7-4881-aa01-e0ffa0564bcc","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.811524481Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"Upstairs opts: Upstairs UUID: b6093006-403d-44fd-8ca9-53b264952a03, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.811586714Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"Crucible stats registered with UUID: b6093006-403d-44fd-8ca9-53b264952a03","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.811616192Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:28.811641473Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.812107956Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.812161612Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.812190289Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.812594349Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.812633457Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336} {"msg":"b6093006-403d-44fd-8ca9-53b264952a03 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.812727693Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.812755258Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"client":"0","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.812786209Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"client":"1","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.812809365Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"client":"2","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.812831639Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.812985844Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.813025142Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:37370)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.813207623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:33169)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.813264887Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"ds_connection connected from Ok(127.0.0.1:33047)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.813442057Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.815619288Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"client":"1","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.815706821Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"client":"2","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.815894673Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"client":"0","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.816666171Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.816737292Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"[0]R flush_numbers[0..12]: [54, 51, 57, 45, 58, 58, 51, 58, 54, 56, 47, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.816972177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"downstairs","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"[0]R generation[0..12]: [31, 30, 33, 28, 34, 34, 30, 34, 31, 32, 28, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817016615Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"downstairs","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817064119Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"downstairs","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"[1]R flush_numbers[0..12]: [54, 51, 57, 45, 58, 58, 51, 58, 54, 56, 47, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817101262Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"downstairs","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"[1]R generation[0..12]: [31, 30, 33, 28, 34, 34, 30, 34, 31, 32, 28, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817137364Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"downstairs","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817176641Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"downstairs","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"[2]R flush_numbers[0..12]: [54, 51, 57, 45, 58, 58, 51, 58, 54, 56, 47, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817213504Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"downstairs","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"[2]R generation[0..12]: [31, 30, 33, 28, 34, 34, 30, 34, 31, 32, 28, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817250177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"downstairs","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817289234Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"downstairs","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"Max found gen is 35","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817321017Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"downstairs","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"Generation requested: 36 >= found:35","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817353822Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"downstairs","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817385044Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"downstairs","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817416406Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817466385Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817496635Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"client":"0","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.81752963Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"client":"1","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817562566Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"client":"2","":"downstairs","session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817599469Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817631061Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"b6093006-403d-44fd-8ca9-53b264952a03 is now active with session: 49372878-50c7-4881-aa01-e0ffa0564bcc","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817666411Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817695819Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336,"session_id":"49372878-50c7-4881-aa01-e0ffa0564bcc"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817726911Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336} {"msg":"Activated sub_volume b6093006-403d-44fd-8ca9-53b264952a03","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:28.817753634Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 402 len 24576 data: 3 3 3 3 3 2 02/30 Write block 437 len 24576 data: 8 9 7 6 5 5 03/30 Write block 365 len 32768 data: 3 3 3 3 3 3 3 3 04/30 Write block 333 len 8192 data: 2 2 05/30 Write block 479 len 20480 data: 5 5 6 6 6 06/30 Write block 36 len 12288 data: 3 3 3 07/30 Read block 528 len 4096 08/30 Read block 54 len 40960 09/30 Flush 10/30 Write block 550 len 28672 data: 3 3 4 3 4 4 5 11/30 Write block 32 len 28672 data: 3 3 3 3 4 4 4 12/30 Read block 303 len 36864 13/30 Read block 284 len 40960 14/30 Read block 172 len 24576 15/30 Read block 398 len 28672 16/30 Write block 404 len 24576 data: 4 4 4 3 2 3 17/30 Read block 566 len 8192 18/30 Write block 503 len 4096 data: 5 19/30 Write block 3 len 20480 data: 3 5 5 5 5 20/30 Write block 566 len 4096 data: 4 21/30 Write block 5 len 24576 data: 6 6 6 4 4 2 22/30 Write block 575 len 12288 data: 4 3 2 23/30 Flush 24/30 Write block 487 len 36864 data: 4 2 2 3 3 3 3 4 4 25/30 Read block 17 len 24576 26/30 Write block 493 len 32768 data: 4 5 5 5 5 5 6 4 27/30 Write block 316 len 8192 data: 3 3 28/30 Write block 430 len 16384 data: 5 4 3 4 29/30 Write block 28 len 16384 data: 2 2 2 2 30/30 Write block 237 len 24576 data: 6 6 6 5 3 3 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Read 2 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 4 Sent Done Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 2 3 32 Done 3 28 27 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:28.844701818Z","hostname":"bmat-EVT22200007-0000ead9","pid":1336} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 36 36 54 60 60 F F F <--- 1 020-039 30 36 36 51 60 60 F T T <--- 2 040-059 33 33 33 57 57 57 F F F 3 060-079 28 28 28 45 45 45 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 30 30 30 51 51 51 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 31 31 31 54 54 54 F F F 9 180-199 32 32 32 56 56 56 F F F 10 200-219 28 28 28 47 47 47 F F F 11 220-239 34 34 34 58 58 58 F T T <--- 12 240-259 33 33 33 57 57 57 F T T <--- 13 260-279 34 34 34 58 58 58 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 28 28 28 47 47 47 F T T <--- 16 320-339 30 36 36 53 59 59 F F F <--- 17 340-359 33 33 33 57 57 57 F F F 18 360-379 34 36 36 58 59 59 T F F <--- 19 380-399 30 30 30 50 50 50 F F F 20 400-419 34 36 36 58 60 60 T F F <--- 21 420-439 34 36 36 58 59 59 T T T <--- 22 440-459 34 36 36 58 59 59 T F F <--- 23 460-479 32 36 36 56 59 59 F F F <--- 24 480-499 31 36 36 54 59 59 F T T <--- 25 500-519 34 36 36 58 60 60 F T T <--- 26 520-539 34 34 34 58 58 58 F F F 27 540-559 34 36 36 58 60 60 F F F <--- 28 560-579 29 36 36 48 60 60 F F F <--- 29 580-599 27 27 27 44 44 44 F F F Max gen: 36, Max flush: 60 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 60 60 60 F F F 1 020-039 37 37 37 61 61 61 F F F 2 040-059 33 33 33 57 57 57 F F F 3 060-079 28 28 28 45 45 45 F F F 4 080-099 34 34 34 58 58 58 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 30 30 30 51 51 51 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 31 31 31 54 54 54 F F F 9 180-199 32 32 32 56 56 56 F F F 10 200-219 28 28 28 47 47 47 F F F 11 220-239 37 37 37 61 61 61 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 34 34 34 58 58 58 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 37 37 37 61 61 61 F F F 16 320-339 36 36 36 59 59 59 F F F 17 340-359 33 33 33 57 57 57 F F F 18 360-379 36 36 36 59 59 59 F F F 19 380-399 30 30 30 50 50 50 F F F 20 400-419 36 36 36 60 60 60 F F F 21 420-439 37 37 37 61 61 61 F F F 22 440-459 36 36 36 59 59 59 F F F 23 460-479 36 36 36 59 59 59 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 37 37 37 61 61 61 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 36 36 36 60 60 60 F F F 28 560-579 36 36 36 60 60 60 F F F 29 580-599 27 27 27 44 44 44 F F F Max gen: 37, Max flush: 61 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.499714849Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.52448007Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.525003216Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.525088445Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.525117022Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.52515656Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344} {"msg":"Crucible 20fd8d71-90e7-4500-a42c-d70f4d38c246 has session id: 8a36455f-eabc-4251-b442-175de2644402","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.526107332Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"Upstairs opts: Upstairs UUID: 20fd8d71-90e7-4500-a42c-d70f4d38c246, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.526177371Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"Crucible stats registered with UUID: 20fd8d71-90e7-4500-a42c-d70f4d38c246","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.526220526Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:33.526251257Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.52664182Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.526764183Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.526826556Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.527301547Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.527350383Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344} {"msg":"20fd8d71-90e7-4500-a42c-d70f4d38c246 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.527572303Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.527689926Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"client":"0","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.527934099Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"client":"1","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.528047624Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"client":"2","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.528086641Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.528125688Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.528164225Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"ds_connection connected from Ok(127.0.0.1:64406)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.528204855Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"ds_connection connected from Ok(127.0.0.1:55687)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.528245526Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"ds_connection connected from Ok(127.0.0.1:37488)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.528285685Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.530125387Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"client":"2","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.530204183Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"client":"0","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.530250916Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"client":"1","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.531227119Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.531280745Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"[0]R flush_numbers[0..12]: [60, 61, 57, 45, 58, 58, 51, 58, 54, 56, 47, 61]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.531437465Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"downstairs","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"[0]R generation[0..12]: [36, 37, 33, 28, 34, 34, 30, 34, 31, 32, 28, 37]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.531486021Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"downstairs","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.531531151Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"downstairs","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"[1]R flush_numbers[0..12]: [60, 61, 57, 45, 58, 58, 51, 58, 54, 56, 47, 61]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.531573424Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"downstairs","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"[1]R generation[0..12]: [36, 37, 33, 28, 34, 34, 30, 34, 31, 32, 28, 37]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.531615007Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"downstairs","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.531662551Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"downstairs","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"[2]R flush_numbers[0..12]: [60, 61, 57, 45, 58, 58, 51, 58, 54, 56, 47, 61]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.531721297Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"downstairs","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"[2]R generation[0..12]: [36, 37, 33, 28, 34, 34, 30, 34, 31, 32, 28, 37]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.53176311Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"downstairs","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.531807788Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"downstairs","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.531856916Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"downstairs","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {Wait for a query_work_queue command to finish before sending IO "msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.531908017Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"downstairs","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"Next flush: 62","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.53194495Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"downstairs","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.531980821Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"":"downstairs","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.532043746Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.532078685Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"client":"0","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.53211659Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"client":"1","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.532154405Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"client":"2","":"downstairs","session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.532192951Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.532233812Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"20fd8d71-90e7-4500-a42c-d70f4d38c246 is now active with session: 8a36455f-eabc-4251-b442-175de2644402","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.532274432Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.532308189Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344,"session_id":"8a36455f-eabc-4251-b442-175de2644402"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.53234386Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344} {"msg":"Activated sub_volume 20fd8d71-90e7-4500-a42c-d70f4d38c246","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:33.53237448Z","hostname":"bmat-EVT22200007-0000ead9","pid":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 1344} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 443 len 32768 data: 4 5 4 5 3 3 3 3 02/30 Read block 24 len 32768 03/30 Flush 04/30 Flush 05/30 Write block 422 len 28672 data: 5 6 7 7 6 5 5 06/30 Read block 493 len 12288 07/30 Read block 588 len 12288 08/30 Flush 09/30 Write block 301 len 16384 data: 3 2 2 3 10/30 Read block 455 len 28672 11/30 Read block 348 len 16384 12/30 Write block 91 len 32768 data: 4 4 4 3 2 3 4 4 13/30 Write block 580 len 28672 data: 3 3 3 3 3 3 3 14/30 Flush 15/30 Write block 35 len 12288 data: 4 5 5 16/30 Read block 120 len 28672 17/30 Read block 112 len 8192 18/30 Read block 347 len 28672 19/30 Flush 20/30 Flush 21/30 Flush 22/30 Write block 187 len 36864 data: 5 6 5 4 6 5 5 5 4 23/30 Read block 366 len 28672 24/30 Write block 436 len 8192 data: 6 9 25/30 Read block 212 len 40960 26/30 Read block 376 len 8192 27/30 Read block 230 len 12288 28/30 Write block 84 len 12288 data: 3 3 4 29/30 Read block 410 len 24576 30/30 Write block 315 len 40960 data: 2 4 4 3 3 3 3 5 4 6 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 4 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 20 1 22 Done 21 2 21 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1007)) Some(JobId(1020)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:33.569316681Z","hostname":"bmat-EVT22200007-0000ead9","pid":1344} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 60 60 60 F F F 1 020-039 38 37 38 66 61 66 F F F <--- 2 040-059 33 33 33 57 57 57 F F F 3 060-079 28 28 28 45 45 45 F F F 4 080-099 38 34 38 65 58 65 T F T <--- 5 100-119 34 34 34 58 58 58 F F F 6 120-139 30 30 30 51 51 51 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 31 31 31 54 54 54 F F F 9 180-199 32 32 32 56 56 56 T F T <--- 10 200-219 28 28 28 47 47 47 F F F 11 220-239 37 37 37 61 61 61 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 34 34 34 58 58 58 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 38 37 38 65 61 65 T T T <--- 16 320-339 36 36 36 59 59 59 T F T <--- 17 340-359 33 33 33 57 57 57 F F F 18 360-379 36 36 36 59 59 59 F F F 19 380-399 30 30 30 50 50 50 F F F 20 400-419 36 36 36 60 60 60 F F F 21 420-439 38 38 38 64 64 64 T F T <--- 22 440-459 38 38 38 62 62 62 F F F 23 460-479 36 36 36 59 59 59 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 37 37 37 61 61 61 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 36 36 36 60 60 60 F F F 28 560-579 36 36 36 60 60 60 F F F 29 580-599 38 27 38 65 44 65 F F F <--- Max gen: 38, Max flush: 66 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 60 60 60 F F F 1 020-039 38 38 38 66 66 66 F F F 2 040-059 33 33 33 57 57 57 F F F 3 060-079 28 28 28 45 45 45 F F F 4 080-099 39 39 39 67 67 67 F F F 5 100-119 34 34 34 58 58 58 F F F 6 120-139 30 30 30 51 51 51 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 31 31 31 54 54 54 F F F 9 180-199 39 39 39 67 67 67 F F F 10 200-219 28 28 28 47 47 47 F F F 11 220-239 37 37 37 61 61 61 F F F 12 240-259 37 37 37 61 61 61 F F F 13 260-279 34 34 34 58 58 58 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 39 39 39 67 67 67 F F F 16 320-339 39 39 39 67 67 67 F F F 17 340-359 33 33 33 57 57 57 F F F 18 360-379 36 36 36 59 59 59 F F F 19 380-399 30 30 30 50 50 50 F F F 20 400-419 36 36 36 60 60 60 F F F 21 420-439 39 39 39 67 67 67 F F F 22 440-459 38 38 38 62 62 62 F F F 23 460-479 36 36 36 59 59 59 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 37 37 37 61 61 61 F F F 26 520-539 34 34 34 58 58 58 F F F 27 540-559 36 36 36 60 60 60 F F F 28 560-579 36 36 36 60 60 60 F F F 29 580-599 38 38 38 65 65 65 F F F Max gen: 39, Max flush: 67 Begin loop 20 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.153406273Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.18074031Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.181352132Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.181459143Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.181481909Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.181520285Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352} {"msg":"Crucible 06fc82ad-d458-431d-b63c-c7f90a292b41 has session id: bbaf6077-6a9f-49fa-8da0-87277689a569","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.182800008Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"Upstairs opts: Upstairs UUID: 06fc82ad-d458-431d-b63c-c7f90a292b41, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.182853785Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"Crucible stats registered with UUID: 06fc82ad-d458-431d-b63c-c7f90a292b41","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.182878944Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:38.182900667Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.183341911Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.183393724Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.183418142Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.183781882Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.183811912Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352} {"msg":"06fc82ad-d458-431d-b63c-c7f90a292b41 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.183970365Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.183996217Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"client":"0","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.184019643Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"client":"1","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.184043039Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"client":"2","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.184136695Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.184234027Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.184267474Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"ds_connection connected from Ok(127.0.0.1:59332)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.184709048Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"ds_connection connected from Ok(127.0.0.1:32936)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.185102647Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"ds_connection connected from Ok(127.0.0.1:36584)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.185201543Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.187121083Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"client":"0","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.187238596Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"client":"1","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.187636534Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"client":"2","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.187918472Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188026145Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"[0]R flush_numbers[0..12]: [60, 66, 57, 45, 67, 58, 51, 58, 54, 67, 47, 61]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188464122Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"downstairs","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"[0]R generation[0..12]: [36, 38, 33, 28, 39, 34, 30, 34, 31, 39, 28, 37]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188502458Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"downstairs","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188534422Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"downstairs","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"[1]R flush_numbers[0..12]: [60, 66, 57, 45, 67, 58, 51, 58, 54, 67, 47, 61]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188562106Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"downstairs","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"[1]R generation[0..12]: [36, 38, 33, 28, 39, 34, 30, 34, 31, 39, 28, 37]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188589751Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"downstairs","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188628368Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"downstairs","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"[2]R flush_numbers[0..12]: [60, 66, 57, 45, 67, 58, 51, 58, 54, 67, 47, 61]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188656493Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"downstairs","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"[2]R generation[0..12]: [36, 38, 33, 28, 39, 34, 30, 34, 31, 39, 28, 37]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.18868513Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"downstairs","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188713556Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"downstairs","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188738115Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"downstairs","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188770599Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"downstairs","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188804246Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"downstairs","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188839616Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"":"downstairs","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188877872Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188910015Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"client":"0","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.18894779Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"client":"1","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.188986477Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"client":"2","":"downstairs","session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:09:38.189024181Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.189068359Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"06fc82ad-d458-431d-b63c-c7f90a292b41 is now active with session: bbaf6077-6a9f-49fa-8da0-87277689a569","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.1891046Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.189138798Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352,"session_id":"bbaf6077-6a9f-49fa-8da0-87277689a569"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.189172675Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352} {"msg":"Activated sub_volume 06fc82ad-d458-431d-b63c-c7f90a292b41","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:38.189204699Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 314 len 12288 data: 2 3 5 02/30 Write block 457 len 16384 data: 7 8 7 7 03/30 Flush 04/30 Write block 335 len 12288 data: 2 3 3 05/30 Read block 209 len 8192 06/30 Write block 554 len 28672 data: 5 5 6 5 5 5 5 07/30 Read block 449 len 16384 08/30 Write block 527 len 8192 data: 3 3 09/30 Write block 508 len 12288 data: 3 3 5 10/30 Read block 29 len 4096 11/30 Write block 344 len 32768 data: 3 2 4 4 4 4 4 4 12/30 Read block 440 len 36864 13/30 Read block 380 len 12288 14/30 Write block 168 len 28672 data: 8 5 4 4 2 2 2 15/30 Read block 13 len 8192 16/30 Write block 2 len 32768 data: 3 4 6 7 7 7 5 5 17/30 Flush 18/30 Write block 533 len 32768 data: 5 6 4 6 5 5 5 5 19/30 Write block 174 len 40960 data: 3 3 5 5 3 3 4 5 5 5 20/30 Write block 228 len 28672 data: 3 3 4 4 5 6 6 21/30 Write block 113 len 32768 data: 7 7 6 6 5 5 4 5 22/30 Write block 402 len 8192 data: 4 4 23/30 Write block 123 len 32768 data: 3 3 2 2 2 2 2 2 24/30 Write block 125 len 20480 data: 3 3 3 3 3 25/30 Flush 26/30 Write block 239 len 16384 data: 7 6 4 4 27/30 Write block 551 len 20480 data: 4 5 4 6 6 28/30 Write block 320 len 12288 data: 4 4 6 29/30 Write block 127 len 32768 data: 4 4 4 3 3 3 4 5 30/30 Read block 433 len 24576 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 3 Done Done Done false 1004 Acked Read 2 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Write 2 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 26 26 Done 27 27 1 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:38.223100518Z","hostname":"bmat-EVT22200007-0000ead9","pid":1352} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 40 40 36 69 69 60 F F F <--- 1 020-039 38 38 38 66 66 66 F F F 2 040-059 33 33 33 57 57 57 F F F 3 060-079 28 28 28 45 45 45 F F F 4 080-099 39 39 39 67 67 67 F F F 5 100-119 40 40 34 70 70 58 F F F <--- 6 120-139 40 40 30 70 70 51 T T F <--- 7 140-159 34 34 34 58 58 58 F F F 8 160-179 40 40 31 70 70 54 F F F <--- 9 180-199 40 40 39 70 70 67 F F F <--- 10 200-219 28 28 28 47 47 47 F F F 11 220-239 40 40 37 70 70 61 T T F <--- 12 240-259 37 37 37 61 61 61 T T F <--- 13 260-279 34 34 34 58 58 58 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 40 40 40 68 68 68 F F F 16 320-339 40 40 39 69 69 67 T T T <--- 17 340-359 40 40 33 69 69 57 F F F <--- 18 360-379 36 36 36 59 59 59 F F F 19 380-399 30 30 30 50 50 50 F F F 20 400-419 40 40 36 70 70 60 F F F <--- 21 420-439 39 39 39 67 67 67 F F F 22 440-459 40 40 40 68 68 68 F F F 23 460-479 40 40 40 68 68 68 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 40 40 37 69 69 61 F F F <--- 26 520-539 40 40 34 70 70 58 F F F <--- 27 540-559 40 40 36 70 70 60 T T F <--- 28 560-579 40 40 36 69 69 60 F F F <--- 29 580-599 38 38 38 65 65 65 F F F Max gen: 40, Max flush: 70 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 40 40 40 69 69 69 F F F 1 020-039 38 38 38 66 66 66 F F F 2 040-059 33 33 33 57 57 57 F F F 3 060-079 28 28 28 45 45 45 F F F 4 080-099 39 39 39 67 67 67 F F F 5 100-119 40 40 40 70 70 70 F F F 6 120-139 41 41 41 71 71 71 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 40 40 40 70 70 70 F F F 9 180-199 40 40 40 70 70 70 F F F 10 200-219 28 28 28 47 47 47 F F F 11 220-239 41 41 41 71 71 71 F F F 12 240-259 41 41 41 71 71 71 F F F 13 260-279 34 34 34 58 58 58 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 40 40 40 68 68 68 F F F 16 320-339 41 41 41 71 71 71 F F F 17 340-359 40 40 40 69 69 69 F F F 18 360-379 36 36 36 59 59 59 F F F 19 380-399 30 30 30 50 50 50 F F F 20 400-419 40 40 40 70 70 70 F F F 21 420-439 39 39 39 67 67 67 F F F 22 440-459 40 40 40 68 68 68 F F F 23 460-479 40 40 40 68 68 68 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 40 40 40 69 69 69 F F F 26 520-539 40 40 40 70 70 70 F F F 27 540-559 41 41 41 71 71 71 F F F 28 560-579 40 40 40 69 69 69 F F F 29 580-599 38 38 38 65 65 65 F F F Max gen: 41, Max flush: 71 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.827548457Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.851886192Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.852601398Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.852694883Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.852724261Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.852768289Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360} {"msg":"Crucible 32d60bcb-c13c-4d58-898e-c4ee01b7c9ab has session id: bac6ae51-8462-486d-9170-47e0ad53ae9c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.853636077Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"Upstairs opts: Upstairs UUID: 32d60bcb-c13c-4d58-898e-c4ee01b7c9ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.853708931Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"Crucible stats registered with UUID: 32d60bcb-c13c-4d58-898e-c4ee01b7c9ab","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.853744602Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:42.853775302Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.854113602Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.854172258Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.854206867Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.854462062Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.854494937Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360} {"msg":"32d60bcb-c13c-4d58-898e-c4ee01b7c9ab active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.854593633Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.854626638Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"client":"0","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.854659743Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"client":"1","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.854692578Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"client":"2","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.854724832Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.85475885Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.854934307Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:51128)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.855369118Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63860)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.855459537Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61749)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.855669813Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.857142659Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"client":"0","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.857189412Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"client":"1","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.857260473Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"client":"2","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.858009345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.858142298Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"[0]R flush_numbers[0..12]: [69, 66, 57, 45, 67, 70, 71, 58, 70, 70, 47, 71]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.858482121Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"downstairs","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"[0]R generation[0..12]: [40, 38, 33, 28, 39, 40, 41, 34, 40, 40, 28, 41]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.858522251Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"downstairs","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.85856206Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"downstairs","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"[1]R flush_numbers[0..12]: [69, 66, 57, 45, 67, 70, 71, 58, 70, 70, 47, 71]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.858598632Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"downstairs","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"[1]R generation[0..12]: [40, 38, 33, 28, 39, 40, 41, 34, 40, 40, 28, 41]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.858634914Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"downstairs","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.85867352Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"downstairs","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"[2]R flush_numbers[0..12]: [69, 66, 57, 45, 67, 70, 71, 58, 70, 70, 47, 71]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.858710053Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"downstairs","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"[2]R generation[0..12]: [40, 38, 33, 28, 39, 40, 41, 34, 40, 40, 28, 41]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.858746024Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"downstairs","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.85878477Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"downstairs","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.858816644Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"downstairs","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.858849388Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"downstairs","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.85888049Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"downstairs","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.858911782Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"":"downstairs","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.858949547Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2026-03-04T01:09:42.858992331Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"client":"0","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.85902694Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"client":"1","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.859059925Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"client":"2","":"downstairs","session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.859093792Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.859125064Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"32d60bcb-c13c-4d58-898e-c4ee01b7c9ab is now active with session: bac6ae51-8462-486d-9170-47e0ad53ae9c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.859160364Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.859191596Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.859216305Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360,"session_id":"bac6ae51-8462-486d-9170-47e0ad53ae9c"} {"msg":"Activated sub_volume 32d60bcb-c13c-4d58-898e-c4ee01b7c9ab","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:42.859247727Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 261 len 12288 data: 5 5 5 02/30 Flush 03/30 Write block 356 len 28672 data: 4 5 7 7 7 7 7 04/30 Write block 456 len 20480 data: 5 8 9 8 8 05/30 Flush 06/30 Write block 529 len 32768 data: 4 4 4 4 6 7 5 7 07/30 Write block 542 len 28672 data: 7 6 4 4 3 4 4 08/30 Read block 64 len 20480 09/30 Write block 60 len 40960 data: 3 3 3 3 5 4 3 2 4 5 10/30 Read block 23 len 40960 11/30 Write block 6 len 4096 data: 8 12/30 Write block 444 len 8192 data: 6 5 13/30 Flush 14/30 Write block 466 len 40960 data: 4 3 3 4 4 3 4 4 5 5 15/30 Write block 438 len 4096 data: 10 16/30 Read block 398 len 28672 17/30 Read block 351 len 36864 18/30 Write block 400 len 36864 data: 4 4 5 5 5 5 5 4 3 19/30 Flush 20/30 Write block 356 len 40960 data: 5 6 8 8 8 8 8 6 5 4 21/30 Read block 67 len 40960 22/30 Read block 123 len 12288 23/30 Read block 217 len 32768 24/30 Read block 371 len 8192 25/30 Read block 536 len 40960 26/30 Write block 335 len 20480 data: 3 4 4 2 2 27/30 Write block 429 len 40960 data: 6 6 5 4 5 5 5 7 10 11 28/30 Write block 180 len 4096 data: 5 29/30 Read block 27 len 28672 30/30 Read block 206 len 4096 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 7 Done Done Done false 1003 Acked Write 5 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 8 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 27 27 Done 28 28 1 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:42.890317922Z","hostname":"bmat-EVT22200007-0000ead9","pid":1360} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 40 74 74 69 F F F <--- 1 020-039 38 38 38 66 66 66 F F F 2 040-059 33 33 33 57 57 57 F F F 3 060-079 42 42 28 74 74 45 F F F <--- 4 080-099 39 39 39 67 67 67 F F F 5 100-119 40 40 40 70 70 70 F F F 6 120-139 41 41 41 71 71 71 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 40 40 40 70 70 70 F F F 9 180-199 40 40 40 70 70 70 T T F <--- 10 200-219 28 28 28 47 47 47 F F F 11 220-239 41 41 41 71 71 71 F F F 12 240-259 41 41 41 71 71 71 F F F 13 260-279 42 42 42 72 72 72 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 40 40 40 68 68 68 F F F 16 320-339 41 41 41 71 71 71 T T F <--- 17 340-359 42 42 40 73 73 69 T T T <--- 18 360-379 42 42 36 73 73 59 T T T <--- 19 380-399 30 30 30 50 50 50 F F F 20 400-419 42 42 40 75 75 70 F F F <--- 21 420-439 42 42 39 75 75 67 T T F <--- 22 440-459 42 42 40 74 74 68 F F F <--- 23 460-479 42 42 40 75 75 68 F F F <--- 24 480-499 37 37 37 61 61 61 F F F 25 500-519 40 40 40 69 69 69 F F F 26 520-539 42 42 40 74 74 70 F F F <--- 27 540-559 42 42 41 74 74 71 F F F <--- 28 560-579 40 40 40 69 69 69 F F F 29 580-599 38 38 38 65 65 65 F F F Max gen: 42, Max flush: 75 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 74 74 74 F F F 1 020-039 38 38 38 66 66 66 F F F 2 040-059 33 33 33 57 57 57 F F F 3 060-079 42 42 42 74 74 74 F F F 4 080-099 39 39 39 67 67 67 F F F 5 100-119 40 40 40 70 70 70 F F F 6 120-139 41 41 41 71 71 71 F F F 7 140-159 34 34 34 58 58 58 F F F 8 160-179 40 40 40 70 70 70 F F F 9 180-199 43 43 43 76 76 76 F F F 10 200-219 28 28 28 47 47 47 F F F 11 220-239 41 41 41 71 71 71 F F F 12 240-259 41 41 41 71 71 71 F F F 13 260-279 42 42 42 72 72 72 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 40 40 40 68 68 68 F F F 16 320-339 43 43 43 76 76 76 F F F 17 340-359 43 43 43 76 76 76 F F F 18 360-379 43 43 43 76 76 76 F F F 19 380-399 30 30 30 50 50 50 F F F 20 400-419 42 42 42 75 75 75 F F F 21 420-439 43 43 43 76 76 76 F F F 22 440-459 42 42 42 74 74 74 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 40 40 40 69 69 69 F F F 26 520-539 42 42 42 74 74 74 F F F 27 540-559 42 42 42 74 74 74 F F F 28 560-579 40 40 40 69 69 69 F F F 29 580-599 38 38 38 65 65 65 F F F Max gen: 43, Max flush: 76 Begin loop 22 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.470231035Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.49726593Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.497823865Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.49790222Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.49792778Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.49796299Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368} {"msg":"Crucible 46a1a35e-f0ae-4e4c-a962-110f7cc6fad1 has session id: 46e1f035-e382-44c8-a205-8e7a637bfe62","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.498834185Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"Upstairs opts: Upstairs UUID: 46a1a35e-f0ae-4e4c-a962-110f7cc6fad1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.498912169Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"Crucible stats registered with UUID: 46a1a35e-f0ae-4e4c-a962-110f7cc6fad1","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.498941117Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:47.498964894Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.499598769Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.499678867Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.499721432Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.500026566Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.500082176Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368} {"msg":"46a1a35e-f0ae-4e4c-a962-110f7cc6fad1 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.500118808Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.500150892Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"client":"0","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.500188486Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"client":"1","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.500230319Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"client":"2","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.500274156Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.500323864Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.500563288Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"ds_connection connected from Ok(127.0.0.1:35991)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.500744217Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"ds_connection connected from Ok(127.0.0.1:36358)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.500807141Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"ds_connection connected from Ok(127.0.0.1:48140)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.500848593Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.502390446Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"client":"2","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.502492258Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"client":"0","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.502889314Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"client":"1","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.503046856Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.503134188Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"[0]R flush_numbers[0..12]: [74, 66, 57, 74, 67, 70, 71, 58, 70, 76, 47, 71]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.503686071Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"downstairs","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"[0]R generation[0..12]: [42, 38, 33, 42, 39, 40, 41, 34, 40, 43, 28, 41]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.503760028Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"downstairs","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.503825156Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"downstairs","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"[1]R flush_numbers[0..12]: [74, 66, 57, 74, 67, 70, 71, 58, 70, 76, 47, 71]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.503884915Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"downstairs","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"[1]R generation[0..12]: [42, 38, 33, 42, 39, 40, 41, 34, 40, 43, 28, 41]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.503943611Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"downstairs","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504031605Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"downstairs","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"[2]R flush_numbers[0..12]: [74, 66, 57, 74, 67, 70, 71, 58, 70, 76, 47, 71]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504091293Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"downstairs","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"[2]R generation[0..12]: [42, 38, 33, 42, 39, 40, 41, 34, 40, 43, 28, 41]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504155511Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"downstairs","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504218255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"downstairs","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-04T01:09:47.504268985Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"downstairs","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504341409Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"downstairs","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504391307Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"downstairs","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504441527Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"":"downstairs","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504494532Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504542206Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"client":"0","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.50459474Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"client":"1","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504648116Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"client":"2","":"downstairs","session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504699327Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504740689Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"46a1a35e-f0ae-4e4c-a962-110f7cc6fad1 is now active with session: 46e1f035-e382-44c8-a205-8e7a637bfe62","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504795758Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504835166Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368,"session_id":"46e1f035-e382-44c8-a205-8e7a637bfe62"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504876308Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368} {"msg":"Activated sub_volume 46a1a35e-f0ae-4e4c-a962-110f7cc6fad1","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:47.504915445Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 577 len 32768 data: 3 3 3 4 4 4 4 4 02/30 Flush 03/30 Write block 129 len 16384 data: 5 4 4 4 04/30 Flush 05/30 Write block 239 len 4096 data: 8 06/30 Write block 345 len 8192 data: 3 5 07/30 Write block 400 len 8192 data: 5 5 08/30 Read block 74 len 24576 09/30 Write block 161 len 32768 data: 5 5 7 7 8 7 8 9 10/30 Write block 124 len 16384 data: 4 4 4 5 11/30 Write block 72 len 8192 data: 3 3 12/30 Write block 133 len 36864 data: 5 6 5 6 6 8 8 7 6 13/30 Read block 388 len 12288 14/30 Flush 15/30 Flush 16/30 Flush 17/30 Write block 336 len 28672 data: 5 5 3 3 3 4 4 18/30 Write block 374 len 32768 data: 3 2 2 2 2 3 2 4 19/30 Write block 320 len 20480 data: 5 5 7 5 7 20/30 Read block 332 len 24576 21/30 Read block 273 len 28672 22/30 Write block 192 len 40960 data: 6 6 6 5 5 5 5 5 6 6 23/30 Read block 459 len 20480 24/30 Write block 551 len 28672 data: 5 6 5 7 7 7 6 25/30 Read block 383 len 24576 26/30 Read block 464 len 24576 27/30 Write block 3 len 20480 data: 5 7 8 9 8 28/30 Read block 451 len 24576 29/30 Read block 493 len 24576 30/30 Write block 570 len 28672 data: 4 4 4 3 4 5 4 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Write 2 Done Sent Done false 1006 Acked Write 2 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Read 6 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:47.537699693Z","hostname":"bmat-EVT22200007-0000ead9","pid":1368} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 74 74 74 T F T <--- 1 020-039 38 38 38 66 66 66 F F F 2 040-059 33 33 33 57 57 57 F F F 3 060-079 44 42 44 79 74 79 F F F <--- 4 080-099 39 39 39 67 67 67 F F F 5 100-119 40 40 40 70 70 70 F F F 6 120-139 44 41 44 79 71 79 F F F <--- 7 140-159 44 34 44 79 58 79 F F F <--- 8 160-179 44 40 44 79 70 79 F F F <--- 9 180-199 43 43 43 76 76 76 T F T <--- 10 200-219 28 28 28 47 47 47 T F T <--- 11 220-239 44 41 44 79 71 79 F F F <--- 12 240-259 41 41 41 71 71 71 F F F 13 260-279 42 42 42 72 72 72 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 40 40 40 68 68 68 F F F 16 320-339 43 43 43 76 76 76 T F T <--- 17 340-359 44 43 44 79 76 79 T F T <--- 18 360-379 43 43 43 76 76 76 T F T <--- 19 380-399 30 30 30 50 50 50 T F T <--- 20 400-419 44 42 44 79 75 79 F F F <--- 21 420-439 43 43 43 76 76 76 F F F 22 440-459 42 42 42 74 74 74 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 40 40 40 69 69 69 F F F 26 520-539 42 42 42 74 74 74 F F F 27 540-559 42 42 42 74 74 74 T F T <--- 28 560-579 44 40 44 77 69 77 T T T <--- 29 580-599 44 38 44 77 65 77 F T F <--- Max gen: 44, Max flush: 79 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 80 80 80 F F F 1 020-039 38 38 38 66 66 66 F F F 2 040-059 33 33 33 57 57 57 F F F 3 060-079 44 44 44 79 79 79 F F F 4 080-099 39 39 39 67 67 67 F F F 5 100-119 40 40 40 70 70 70 F F F 6 120-139 44 44 44 79 79 79 F F F 7 140-159 44 44 44 79 79 79 F F F 8 160-179 44 44 44 79 79 79 F F F 9 180-199 45 45 45 80 80 80 F F F 10 200-219 45 45 45 80 80 80 F F F 11 220-239 44 44 44 79 79 79 F F F 12 240-259 41 41 41 71 71 71 F F F 13 260-279 42 42 42 72 72 72 F F F 14 280-299 34 34 34 58 58 58 F F F 15 300-319 40 40 40 68 68 68 F F F 16 320-339 45 45 45 80 80 80 F F F 17 340-359 45 45 45 80 80 80 F F F 18 360-379 45 45 45 80 80 80 F F F 19 380-399 45 45 45 80 80 80 F F F 20 400-419 44 44 44 79 79 79 F F F 21 420-439 43 43 43 76 76 76 F F F 22 440-459 42 42 42 74 74 74 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 40 40 40 69 69 69 F F F 26 520-539 42 42 42 74 74 74 F F F 27 540-559 45 45 45 80 80 80 F F F 28 560-579 45 45 45 80 80 80 F F F 29 580-599 44 44 44 77 77 77 F F F Max gen: 45, Max flush: 80 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.238499175Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.261419634Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.261918502Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.262020564Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.26204872Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.262092376Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376} {"msg":"Crucible 0b8665be-2167-4a9a-a39c-f68a24c886f7 has session id: 6cecad86-10ef-45da-a411-db66856b391b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.263022517Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"Upstairs opts: Upstairs UUID: 0b8665be-2167-4a9a-a39c-f68a24c886f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.263088859Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"Crucible stats registered with UUID: 0b8665be-2167-4a9a-a39c-f68a24c886f7","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.263123257Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:52.263153717Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.263652585Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.263733826Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.263761801Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.26379143Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.263884284Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376} {"msg":"0b8665be-2167-4a9a-a39c-f68a24c886f7 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.26405926Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.264095171Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"client":"0","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.264409293Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"client":"1","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.264451557Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"client":"2","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.264476667Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.264501937Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.264527678Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45708)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.264553929Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49303)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.264580692Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49691)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.264606844Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.266121884Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"client":"0","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.266610301Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"client":"1","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.266840757Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"client":"2","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.266935304Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.26737739Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"[0]R flush_numbers[0..12]: [80, 66, 57, 79, 67, 70, 79, 79, 79, 80, 80, 79]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.267579159Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"downstairs","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"[0]R generation[0..12]: [45, 38, 33, 44, 39, 40, 44, 44, 44, 45, 45, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.267643176Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"downstairs","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.267693095Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"downstairs","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"[1]R flush_numbers[0..12]: [80, 66, 57, 79, 67, 70, 79, 79, 79, 80, 80, 79]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.267739547Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"downstairs","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"[1]R generation[0..12]: [45, 38, 33, 44, 39, 40, 44, 44, 44, 45, 45, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.267776981Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"downstairs","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.267819846Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"downstairs","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"[2]R flush_numbers[0..12]: [80, 66, 57, 79, 67, 70, 79, 79, 79, 80, 80, 79]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.267867259Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"downstairs","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"[2]R generation[0..12]: [45, 38, 33, 44, 39, 40, 44, 44, 44, 45, 45, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.267905205Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"downstairs","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.267944753Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"downstairs","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.26797876Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"downstairs","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.268013258Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"downstairs","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.268043067Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"downstairs","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.268072165Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"":"downstairs","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.268101653Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.268129198Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"client":"0","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.268168977Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"client":"1","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.268205108Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"client":"2","":"downstairs","session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.268243314Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:09:52.268275207Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"0b8665be-2167-4a9a-a39c-f68a24c886f7 is now active with session: 6cecad86-10ef-45da-a411-db66856b391b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.268334985Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.268366808Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376,"session_id":"6cecad86-10ef-45da-a411-db66856b391b"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.268396076Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376} {"msg":"Activated sub_volume 0b8665be-2167-4a9a-a39c-f68a24c886f7","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:52.268420865Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 68 len 12288 data: 5 6 5 02/30 Write block 320 len 20480 data: 6 6 8 6 8 03/30 Write block 105 len 16384 data: 4 3 3 3 04/30 Write block 404 len 40960 data: 6 6 6 5 4 4 3 3 2 2 05/30 Read block 50 len 36864 06/30 Write block 548 len 32768 data: 5 4 4 6 7 6 8 8 07/30 Flush 08/30 Write block 362 len 16384 data: 9 7 6 5 09/30 Write block 260 len 40960 data: 5 6 6 6 5 3 3 3 5 5 10/30 Read block 233 len 8192 11/30 Write block 240 len 8192 data: 7 5 12/30 Read block 312 len 16384 13/30 Write block 276 len 28672 data: 3 3 3 4 4 4 4 14/30 Read block 200 len 28672 15/30 Read block 275 len 8192 16/30 Read block 219 len 24576 17/30 Read block 8 len 28672 18/30 Read block 499 len 16384 19/30 Flush 20/30 Write block 55 len 12288 data: 2 3 3 21/30 Read block 502 len 28672 22/30 Read block 59 len 40960 23/30 Read block 564 len 24576 24/30 Write block 340 len 4096 data: 4 25/30 Write block 263 len 16384 data: 7 6 4 4 26/30 Write block 205 len 12288 data: 5 5 6 27/30 Write block 102 len 4096 data: 3 28/30 Write block 27 len 24576 data: 3 3 3 3 3 4 29/30 Write block 93 len 32768 data: 5 4 3 4 5 5 6 5 30/30 Write block 578 len 32768 data: 4 4 5 5 5 5 5 4 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Sent Done false 1001 Acked Write 5 Done Sent Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:52.2963041Z","hostname":"bmat-EVT22200007-0000ead9","pid":1376} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 80 80 80 F F F 1 020-039 38 38 38 66 66 66 T F T <--- 2 040-059 33 33 33 57 57 57 T F T <--- 3 060-079 46 44 46 81 79 81 F F F <--- 4 080-099 39 39 39 67 67 67 T F T <--- 5 100-119 46 40 46 81 70 81 T F T <--- 6 120-139 44 44 44 79 79 79 F F F 7 140-159 44 44 44 79 79 79 F F F 8 160-179 44 44 44 79 79 79 F F F 9 180-199 45 45 45 80 80 80 F F F 10 200-219 45 45 45 80 80 80 T F T <--- 11 220-239 44 44 44 79 79 79 F F F 12 240-259 46 41 46 82 71 82 F F F <--- 13 260-279 46 42 46 82 72 82 T F T <--- 14 280-299 46 34 46 82 58 82 F F F <--- 15 300-319 40 40 40 68 68 68 F F F 16 320-339 46 45 46 81 80 81 F F F <--- 17 340-359 45 45 45 80 80 80 T F T <--- 18 360-379 46 45 46 82 80 82 F F F <--- 19 380-399 45 45 45 80 80 80 F F F 20 400-419 46 44 46 81 79 81 F F F <--- 21 420-439 43 43 43 76 76 76 F F F 22 440-459 42 42 42 74 74 74 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 40 40 40 69 69 69 F F F 26 520-539 42 42 42 74 74 74 F F F 27 540-559 46 45 46 81 80 81 F F F <--- 28 560-579 45 45 45 80 80 80 T F T <--- 29 580-599 44 44 44 77 77 77 T F T <--- Max gen: 46, Max flush: 82 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 80 80 80 F F F 1 020-039 47 47 47 83 83 83 F F F 2 040-059 47 47 47 83 83 83 F F F 3 060-079 46 46 46 81 81 81 F F F 4 080-099 47 47 47 83 83 83 F F F 5 100-119 47 47 47 83 83 83 F F F 6 120-139 44 44 44 79 79 79 F F F 7 140-159 44 44 44 79 79 79 F F F 8 160-179 44 44 44 79 79 79 F F F 9 180-199 45 45 45 80 80 80 F F F 10 200-219 47 47 47 83 83 83 F F F 11 220-239 44 44 44 79 79 79 F F F 12 240-259 46 46 46 82 82 82 F F F 13 260-279 47 47 47 83 83 83 F F F 14 280-299 46 46 46 82 82 82 F F F 15 300-319 40 40 40 68 68 68 F F F 16 320-339 46 46 46 81 81 81 F F F 17 340-359 47 47 47 83 83 83 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 45 45 45 80 80 80 F F F 20 400-419 46 46 46 81 81 81 F F F 21 420-439 43 43 43 76 76 76 F F F 22 440-459 42 42 42 74 74 74 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 40 40 40 69 69 69 F F F 26 520-539 42 42 42 74 74 74 F F F 27 540-559 46 46 46 81 81 81 F F F 28 560-579 47 47 47 83 83 83 F F F 29 580-599 47 47 47 83 83 83 F F F Max gen: 47, Max flush: 83 Begin loop 24 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.921874768Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:56.944442149Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.944556535Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.967677652Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.968220015Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.968309252Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.968338029Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.968382838Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384} {"msg":"Crucible 21982009-ec51-4a76-996b-e543e823d7b2 has session id: b63d8499-a81c-4b56-98f3-1c5e1e550c06","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.969541792Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"Upstairs opts: Upstairs UUID: 21982009-ec51-4a76-996b-e543e823d7b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.96964175Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"Crucible stats registered with UUID: 21982009-ec51-4a76-996b-e543e823d7b2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.969670557Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:56.969694435Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.970134476Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.970204925Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.97023241Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.970258271Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.970367627Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384} {"msg":"21982009-ec51-4a76-996b-e543e823d7b2 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.970396885Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.97041949Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"client":"0","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.9704446Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"client":"1","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.97046972Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"client":"2","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.970493857Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.970610178Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.970640929Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"ds_connection connected from Ok(127.0.0.1:52772)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.970855473Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"ds_connection connected from Ok(127.0.0.1:54044)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.971053736Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"ds_connection connected from Ok(127.0.0.1:41864)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.971090549Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.972701398Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"client":"2","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.972776176Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"client":"0","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.972985741Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"client":"1","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.97350055Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.973631239Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"[0]R flush_numbers[0..12]: [80, 83, 83, 81, 83, 83, 79, 79, 79, 80, 83, 79]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.973800434Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"downstairs","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"[0]R generation[0..12]: [45, 47, 47, 46, 47, 47, 44, 44, 44, 45, 47, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.973832888Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"downstairs","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.973862858Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"downstairs","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"[1]R flush_numbers[0..12]: [80, 83, 83, 81, 83, 83, 79, 79, 79, 80, 83, 79]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.973890412Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"downstairs","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"[1]R generation[0..12]: [45, 47, 47, 46, 47, 47, 44, 44, 44, 45, 47, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.973917856Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"downstairs","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.973947134Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"downstairs","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"[2]R flush_numbers[0..12]: [80, 83, 83, 81, 83, 83, 79, 79, 79, 80, 83, 79]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.973981462Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"downstairs","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"[2]R generation[0..12]: [45, 47, 47, 46, 47, 47, 44, 44, 44, 45, 47, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.974009348Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"downstairs","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.974038756Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"downstairs","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.974062984Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"downstairs","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.974088063Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"downstairs","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.97411181Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"downstairs","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.974135497Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"":"downstairs","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.974160397Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.974183262Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"client":"0","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.974208492Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"client":"1","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.97423292Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"client":"2","":"downstairs","session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.974258351Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.974282138Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"21982009-ec51-4a76-996b-e543e823d7b2 is now active with session: b63d8499-a81c-4b56-98f3-1c5e1e550c06","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.97430867Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.974330523Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384,"session_id":"b63d8499-a81c-4b56-98f3-1c5e1e550c06"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.974353749Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384} {"msg":"Activated sub_volume 21982009-ec51-4a76-996b-e543e823d7b2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:09:56.974377587Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 244 len 40960 data: 4 4 5 5 4 7 7 3 4 3 02/30 Flush 03/30 Write block 394 len 24576 data: 4 3 3 3 3 3 04/30 Read block 162 len 28672 05/30 Write block 193 len 40960 data: 7 7 6 6 6 6 6 7 7 6 06/30 Read block 538 len 16384 07/30 Read block 246 len 16384 08/30 Write block 256 len 8192 data: 3 4 09/30 Read block 511 len 36864 10/30 Write block 109 len 20480 data: 2 2 3 3 8 11/30 Read block 517 len 28672 12/30 Write block 403 len 12288 data: 6 7 7 13/30 Read block 504 len 12288 14/30 Read block 53 len 40960 15/30 Write block 268 len 20480 data: 6 6 5 5 5 16/30 Read block 330 len 16384 17/30 Write block 187 len 28672 data: 6 7 6 5 7 7 8 18/30 Flush 19/30 Write block 51 len 28672 data: 5 4 4 4 3 4 4 20/30 Read block 164 len 4096 21/30 Read block 372 len 28672 22/30 Write block 507 len 12288 data: 3 4 4 23/30 Write block 260 len 16384 data: 6 7 7 8 24/30 Read block 140 len 16384 25/30 Write block 260 len 32768 data: 7 8 8 9 7 5 5 4 26/30 Read block 141 len 24576 27/30 Write block 401 len 32768 data: 6 6 7 8 8 7 6 5 28/30 Read block 293 len 28672 29/30 Write block 286 len 28672 data: 3 3 4 4 4 7 7 30/30 Read block 457 len 4096 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 6 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Write 7 Sent Sent Done false 1029 Acked Read 1 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 2 0 30 Done 0 26 28 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:09:57.014470255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1384} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 80 80 80 F F F 1 020-039 47 47 47 83 83 83 F F F 2 040-059 47 47 47 83 83 83 F T T <--- 3 060-079 46 46 46 81 81 81 F F F 4 080-099 47 47 47 83 83 83 F F F 5 100-119 47 48 48 83 85 85 F F F <--- 6 120-139 44 44 44 79 79 79 F F F 7 140-159 44 44 44 79 79 79 F F F 8 160-179 44 44 44 79 79 79 F F F 9 180-199 45 48 48 80 85 85 F F F <--- 10 200-219 47 48 48 83 85 85 F F F <--- 11 220-239 44 44 44 79 79 79 F F F 12 240-259 48 48 48 84 85 85 F F F <--- 13 260-279 47 48 48 83 85 85 F T T <--- 14 280-299 46 46 46 82 82 82 F T T <--- 15 300-319 40 40 40 68 68 68 F F F 16 320-339 46 46 46 81 81 81 F F F 17 340-359 47 47 47 83 83 83 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 45 48 48 80 85 85 F F F <--- 20 400-419 46 48 48 81 85 85 F T T <--- 21 420-439 43 43 43 76 76 76 F F F 22 440-459 42 42 42 74 74 74 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 40 40 40 69 69 69 F T T <--- 26 520-539 42 42 42 74 74 74 F F F 27 540-559 46 46 46 81 81 81 F F F 28 560-579 47 47 47 83 83 83 F F F 29 580-599 47 47 47 83 83 83 F F F Max gen: 48, Max flush: 85 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 80 80 80 F F F 1 020-039 47 47 47 83 83 83 F F F 2 040-059 49 49 49 86 86 86 F F F 3 060-079 46 46 46 81 81 81 F F F 4 080-099 47 47 47 83 83 83 F F F 5 100-119 48 48 48 85 85 85 F F F 6 120-139 44 44 44 79 79 79 F F F 7 140-159 44 44 44 79 79 79 F F F 8 160-179 44 44 44 79 79 79 F F F 9 180-199 48 48 48 85 85 85 F F F 10 200-219 48 48 48 85 85 85 F F F 11 220-239 44 44 44 79 79 79 F F F 12 240-259 48 48 48 85 85 85 F F F 13 260-279 49 49 49 86 86 86 F F F 14 280-299 49 49 49 86 86 86 F F F 15 300-319 40 40 40 68 68 68 F F F 16 320-339 46 46 46 81 81 81 F F F 17 340-359 47 47 47 83 83 83 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 48 48 48 85 85 85 F F F 20 400-419 49 49 49 86 86 86 F F F 21 420-439 43 43 43 76 76 76 F F F 22 440-459 42 42 42 74 74 74 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 49 49 49 86 86 86 F F F 26 520-539 42 42 42 74 74 74 F F F 27 540-559 46 46 46 81 81 81 F F F 28 560-579 47 47 47 83 83 83 F F F 29 580-599 47 47 47 83 83 83 F F F Max gen: 49, Max flush: 86 Begin loop 25 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.598889419Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:01.623798866Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.623902732Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.64657504Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.647122013Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.647206721Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.647238143Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.647281869Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392} {"msg":"Crucible f5448f96-f8be-42ba-aaed-4d5f6e3bc51f has session id: 370aa7c9-5c18-4879-8b90-b858f6b05d10","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.648611482Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"Upstairs opts: Upstairs UUID: f5448f96-f8be-42ba-aaed-4d5f6e3bc51f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.648715217Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"Crucible stats registered with UUID: f5448f96-f8be-42ba-aaed-4d5f6e3bc51f","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.648748994Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:01.648772611Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.649212853Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.649267841Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.64930271Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.649521193Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.649552315Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392} {"msg":"f5448f96-f8be-42ba-aaed-4d5f6e3bc51f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.649675438Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.649708033Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"client":"0","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.649733443Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"client":"1","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.649758052Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"client":"2","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.649782871Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.649816408Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.649847269Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"ds_connection connected from Ok(127.0.0.1:50879)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.650577795Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"ds_connection connected from Ok(127.0.0.1:36952)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.650687602Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"ds_connection connected from Ok(127.0.0.1:34018)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.650728453Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.652344132Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"client":"2","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.652473408Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"client":"0","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.652513507Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"client":"1","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653400754Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653439771Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"[0]R flush_numbers[0..12]: [80, 83, 86, 81, 83, 85, 79, 79, 79, 85, 85, 79]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653467837Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"downstairs","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"[0]R generation[0..12]: [45, 47, 49, 46, 47, 48, 44, 44, 44, 48, 48, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653498508Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"downstairs","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653530651Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"downstairs","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"[1]R flush_numbers[0..12]: [80, 83, 86, 81, 83, 85, 79, 79, 79, 85, 85, 79]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653560651Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"downstairs","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"[1]R generation[0..12]: [45, 47, 49, 46, 47, 48, 44, 44, 44, 48, 48, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653593345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"downstairs","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653632563Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"downstairs","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"[2]R flush_numbers[0..12]: [80, 83, 86, 81, 83, 85, 79, 79, 79, 85, 85, 79]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653662743Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"downstairs","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"[2]R generation[0..12]: [45, 47, 49, 46, 47, 48, 44, 44, 44, 48, 48, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653692542Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"downstairs","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653724515Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"downstairs","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653750066Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"downstairs","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-04T01:10:01.653776799Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"downstairs","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653815024Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"downstairs","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653840695Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"":"downstairs","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653876155Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653901355Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"client":"0","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653935272Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"client":"1","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.653962065Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"client":"2","":"downstairs","session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.654014118Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.654041553Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"f5448f96-f8be-42ba-aaed-4d5f6e3bc51f is now active with session: 370aa7c9-5c18-4879-8b90-b858f6b05d10","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.65407049Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.654101501Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392,"session_id":"370aa7c9-5c18-4879-8b90-b858f6b05d10"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.654128565Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392} {"msg":"Activated sub_volume f5448f96-f8be-42ba-aaed-4d5f6e3bc51f","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:01.654150659Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 503 len 8192 data: 6 5 02/30 Write block 397 len 24576 data: 4 4 4 6 7 7 03/30 Write block 281 len 40960 data: 5 5 4 4 3 4 4 5 5 5 04/30 Read block 203 len 4096 05/30 Read block 42 len 20480 06/30 Write block 254 len 32768 data: 3 4 4 5 4 5 8 9 07/30 Write block 385 len 32768 data: 4 3 3 3 3 3 4 5 08/30 Flush 09/30 Write block 284 len 32768 data: 5 4 5 5 6 6 6 8 10/30 Write block 507 len 24576 data: 4 5 5 6 5 5 11/30 Write block 505 len 8192 data: 4 4 12/30 Read block 499 len 20480 13/30 Read block 539 len 24576 14/30 Read block 514 len 16384 15/30 Write block 448 len 8192 data: 4 4 16/30 Read block 316 len 28672 17/30 Write block 20 len 28672 data: 4 5 6 4 4 2 2 18/30 Read block 255 len 8192 19/30 Read block 498 len 12288 20/30 Read block 8 len 40960 21/30 Read block 129 len 24576 22/30 Read block 547 len 36864 23/30 Flush 24/30 Write block 241 len 36864 data: 6 5 4 5 5 6 6 5 8 25/30 Read block 532 len 12288 26/30 Write block 503 len 12288 data: 7 6 5 27/30 Write block 545 len 12288 data: 5 4 5 28/30 Write block 568 len 32768 data: 4 4 5 5 5 4 5 6 29/30 Write block 115 len 36864 data: 7 7 6 6 5 6 3 4 4 30/30 Read block 376 len 28672 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 8 Done Done Done false 1009 Acked Write 6 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Write 9 Sent Sent Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 16 2 0 18 Done 6 20 22 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:01.687935177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1392} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 80 80 80 F F F 1 020-039 47 50 50 83 88 88 F F F <--- 2 040-059 49 49 49 86 86 86 F F F 3 060-079 46 46 46 81 81 81 F F F 4 080-099 47 47 47 83 83 83 F F F 5 100-119 48 48 48 85 85 85 F T T <--- 6 120-139 44 44 44 79 79 79 F T T <--- 7 140-159 44 44 44 79 79 79 F F F 8 160-179 44 44 44 79 79 79 F F F 9 180-199 48 48 48 85 85 85 F F F 10 200-219 48 48 48 85 85 85 F F F 11 220-239 44 44 44 79 79 79 F F F 12 240-259 50 50 50 87 87 87 F T T <--- 13 260-279 50 50 50 87 87 87 F F F 14 280-299 50 50 50 87 88 88 T F F <--- 15 300-319 40 40 40 68 68 68 F F F 16 320-339 46 46 46 81 81 81 F F F 17 340-359 47 47 47 83 83 83 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 50 50 50 87 87 87 F F F 20 400-419 50 50 50 87 87 87 F F F 21 420-439 43 43 43 76 76 76 F F F 22 440-459 42 50 50 74 88 88 F F F <--- 23 460-479 42 42 42 75 75 75 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 50 50 50 87 88 88 T T T <--- 26 520-539 42 42 42 74 74 74 F F F 27 540-559 46 46 46 81 81 81 F T T <--- 28 560-579 47 47 47 83 83 83 F T T <--- 29 580-599 47 47 47 83 83 83 F F F Max gen: 50, Max flush: 88 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 80 80 80 F F F 1 020-039 50 50 50 88 88 88 F F F 2 040-059 49 49 49 86 86 86 F F F 3 060-079 46 46 46 81 81 81 F F F 4 080-099 47 47 47 83 83 83 F F F 5 100-119 51 51 51 89 89 89 F F F 6 120-139 51 51 51 89 89 89 F F F 7 140-159 44 44 44 79 79 79 F F F 8 160-179 44 44 44 79 79 79 F F F 9 180-199 48 48 48 85 85 85 F F F 10 200-219 48 48 48 85 85 85 F F F 11 220-239 44 44 44 79 79 79 F F F 12 240-259 51 51 51 89 89 89 F F F 13 260-279 50 50 50 87 87 87 F F F 14 280-299 50 50 50 88 88 88 F F F 15 300-319 40 40 40 68 68 68 F F F 16 320-339 46 46 46 81 81 81 F F F 17 340-359 47 47 47 83 83 83 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 50 50 50 87 87 87 F F F 20 400-419 50 50 50 87 87 87 F F F 21 420-439 43 43 43 76 76 76 F F F 22 440-459 50 50 50 88 88 88 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 51 51 51 89 89 89 F F F 26 520-539 42 42 42 74 74 74 F F F 27 540-559 51 51 51 89 89 89 F F F 28 560-579 51 51 51 89 89 89 F F F 29 580-599 47 47 47 83 83 83 F F F Max gen: 51, Max flush: 89 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.287969146Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:06.306668357Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.306761342Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.325158916Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.325732051Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.325813222Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.325839043Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.325877749Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400} {"msg":"Crucible 280c44ca-21f7-4ceb-a3aa-84f76e5815de has session id: 87d74c29-72c9-4e1f-98a3-aa4f104ca186","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.326950704Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"Upstairs opts: Upstairs UUID: 280c44ca-21f7-4ceb-a3aa-84f76e5815de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.327092014Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"Crucible stats registered with UUID: 280c44ca-21f7-4ceb-a3aa-84f76e5815de","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.327141411Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:06.327182663Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.327484771Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.327550652Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.327586032Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.327916947Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.327950884Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400} {"msg":"280c44ca-21f7-4ceb-a3aa-84f76e5815de active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.328078196Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.328111422Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"client":"0","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.32814092Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"client":"1","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.328167824Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"client":"2","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.328194767Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.328310736Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.328349553Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"ds_connection connected from Ok(127.0.0.1:46853)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.328666611Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"ds_connection connected from Ok(127.0.0.1:45379)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.328763152Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"ds_connection connected from Ok(127.0.0.1:33306)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.328804434Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.330651862Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"client":"1","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.3308937Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"client":"2","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.331201279Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"client":"0","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.331533287Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.331560551Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"[0]R flush_numbers[0..12]: [80, 88, 86, 81, 83, 89, 89, 79, 79, 85, 85, 79]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332368731Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"downstairs","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"[0]R generation[0..12]: [45, 50, 49, 46, 47, 51, 51, 44, 44, 48, 48, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332401646Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"downstairs","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332427186Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"downstairs","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"[1]R flush_numbers[0..12]: [80, 88, 86, 81, 83, 89, 89, 79, 79, 85, 85, 79]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332450883Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"downstairs","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"[1]R generation[0..12]: [45, 50, 49, 46, 47, 51, 51, 44, 44, 48, 48, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332474811Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"downstairs","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.33249986Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"downstairs","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"[2]R flush_numbers[0..12]: [80, 88, 86, 81, 83, 89, 89, 79, 79, 85, 85, 79]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332530561Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"downstairs","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"[2]R generation[0..12]: [45, 50, 49, 46, 47, 51, 51, 44, 44, 48, 48, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332554589Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"downstairs","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332579508Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"downstairs","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332599708Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"downstairs","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.3326208Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"downstairs","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.33264097Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"downstairs","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332660999Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"":"downstairs","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332681831Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332701109Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"client":"0","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332722231Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"client":"1","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332742952Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"client":"2","":"downstairs","session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332764424Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332784173Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"280c44ca-21f7-4ceb-a3aa-84f76e5815de is now active with session: 87d74c29-72c9-4e1f-98a3-aa4f104ca186","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332806417Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332825585Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400,"session_id":"87d74c29-72c9-4e1f-98a3-aa4f104ca186"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332845475Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400} {"msg":"Activated sub_volume 280c44ca-21f7-4ceb-a3aa-84f76e5815de","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:06.332866817Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 456 len 20480 data: 6 9 10 9 9 02/30 Read block 231 len 24576 03/30 Write block 112 len 40960 data: 4 9 8 8 8 7 7 6 7 4 04/30 Write block 572 len 40960 data: 6 5 6 7 5 4 5 5 6 6 05/30 Write block 79 len 24576 data: 2 2 3 4 3 4 06/30 Write block 306 len 36864 data: 3 5 5 5 6 7 5 4 3 07/30 Read block 551 len 4096 08/30 Read block 126 len 32768 09/30 Read block 374 len 40960 10/30 Read block 446 len 36864 11/30 Write block 97 len 4096 data: 6 12/30 Write block 214 len 20480 data: 2 2 3 4 4 13/30 Write block 522 len 32768 data: 4 4 4 4 4 4 4 5 14/30 Flush 15/30 Write block 245 len 4096 data: 6 16/30 Read block 296 len 32768 17/30 Read block 222 len 28672 18/30 Write block 320 len 8192 data: 7 7 19/30 Write block 315 len 20480 data: 4 6 5 4 4 20/30 Read block 306 len 20480 21/30 Write block 147 len 8192 data: 7 7 22/30 Flush 23/30 Write block 228 len 28672 data: 4 4 5 5 6 7 7 24/30 Flush 25/30 Write block 7 len 4096 data: 9 26/30 Read block 515 len 16384 27/30 Read block 182 len 36864 28/30 Write block 405 len 24576 data: 9 8 7 6 5 4 29/30 Write block 323 len 32768 data: 7 9 5 4 4 3 3 3 30/30 Write block 584 len 28672 data: 6 5 4 2 2 2 3 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Sent Done Done false 1001 Acked Read 6 Sent Done Done false 1002 Acked Write 10 Sent Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Write 6 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Write 2 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Write 8 Sent Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 1 33 Done 0 28 29 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:06.368500647Z","hostname":"bmat-EVT22200007-0000ead9","pid":1400} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 80 80 80 F T T <--- 1 020-039 50 50 50 88 88 88 F F F 2 040-059 49 49 49 86 86 86 F F F 3 060-079 46 52 52 81 90 90 F F F <--- 4 080-099 47 52 52 83 90 90 F F F <--- 5 100-119 51 52 52 89 90 90 F F F <--- 6 120-139 51 52 52 89 90 90 F F F <--- 7 140-159 44 52 52 79 91 91 F F F <--- 8 160-179 44 44 44 79 79 79 F F F 9 180-199 48 48 48 85 85 85 F F F 10 200-219 48 52 52 85 90 90 F F F <--- 11 220-239 44 52 52 79 92 92 F F F <--- 12 240-259 51 52 52 89 91 91 F F F <--- 13 260-279 50 50 50 87 87 87 F F F 14 280-299 50 50 50 88 88 88 F F F 15 300-319 40 52 52 68 91 91 F F F <--- 16 320-339 46 52 52 81 91 91 F T T <--- 17 340-359 47 47 47 83 83 83 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 50 50 50 87 87 87 F F F 20 400-419 50 50 50 87 87 87 F T T <--- 21 420-439 43 43 43 76 76 76 F F F 22 440-459 50 52 52 88 90 90 F F F <--- 23 460-479 42 52 52 75 90 90 F F F <--- 24 480-499 37 37 37 61 61 61 F F F 25 500-519 51 51 51 89 89 89 F F F 26 520-539 42 52 52 74 90 90 F F F <--- 27 540-559 51 51 51 89 89 89 F F F 28 560-579 51 52 52 89 90 90 F F F <--- 29 580-599 47 52 52 83 90 90 F T T <--- Max gen: 52, Max flush: 92 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 93 93 93 F F F 1 020-039 50 50 50 88 88 88 F F F 2 040-059 49 49 49 86 86 86 F F F 3 060-079 52 52 52 90 90 90 F F F 4 080-099 52 52 52 90 90 90 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 52 52 52 90 90 90 F F F 7 140-159 52 52 52 91 91 91 F F F 8 160-179 44 44 44 79 79 79 F F F 9 180-199 48 48 48 85 85 85 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 52 52 52 92 92 92 F F F 12 240-259 52 52 52 91 91 91 F F F 13 260-279 50 50 50 87 87 87 F F F 14 280-299 50 50 50 88 88 88 F F F 15 300-319 52 52 52 91 91 91 F F F 16 320-339 53 53 53 93 93 93 F F F 17 340-359 47 47 47 83 83 83 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 50 50 50 87 87 87 F F F 20 400-419 53 53 53 93 93 93 F F F 21 420-439 43 43 43 76 76 76 F F F 22 440-459 52 52 52 90 90 90 F F F 23 460-479 52 52 52 90 90 90 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 51 51 51 89 89 89 F F F 26 520-539 52 52 52 90 90 90 F F F 27 540-559 51 51 51 89 89 89 F F F 28 560-579 52 52 52 90 90 90 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 53, Max flush: 93 Begin loop 27 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.059979134Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.086431566Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.087191771Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.087313813Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.087343451Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.087386968Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408} {"msg":"Crucible 301b6b19-1e81-40d7-8d14-287274ddd923 has session id: 86588faa-b28e-40b5-b986-676d627e853e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.088396206Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"Upstairs opts: Upstairs UUID: 301b6b19-1e81-40d7-8d14-287274ddd923, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.088465703Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"Crucible stats registered with UUID: 301b6b19-1e81-40d7-8d14-287274ddd923","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.088501093Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:11.088531894Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.089050862Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.089113636Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.089148325Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.089361667Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.089399121Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408} {"msg":"301b6b19-1e81-40d7-8d14-287274ddd923 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.089538788Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.089572144Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"client":"0","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.0896054Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"client":"1","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.089638205Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"client":"2","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.089670909Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.089819343Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.089857198Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"ds_connection connected from Ok(127.0.0.1:51609)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.09008558Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"ds_connection connected from Ok(127.0.0.1:40889)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.090128205Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34918)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.090648144Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.092254404Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"client":"1","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.092329513Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"client":"2","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.092970031Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"client":"0","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.093052865Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.093168685Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"[0]R flush_numbers[0..12]: [93, 88, 86, 90, 90, 90, 90, 91, 79, 85, 90, 92]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.093637362Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"downstairs","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"[0]R generation[0..12]: [53, 50, 49, 52, 52, 52, 52, 52, 44, 48, 52, 52]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.093684015Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"downstairs","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.093716499Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"downstairs","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"[1]R flush_numbers[0..12]: [93, 88, 86, 90, 90, 90, 90, 91, 79, 85, 90, 92]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.093746168Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"downstairs","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"[1]R generation[0..12]: [53, 50, 49, 52, 52, 52, 52, 52, 44, 48, 52, 52]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.093775286Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"downstairs","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.093806287Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"downstairs","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"[2]R flush_numbers[0..12]: [93, 88, 86, 90, 90, 90, 90, 91, 79, 85, 90, 92]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.093834022Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"downstairs","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"[2]R generation[0..12]: [53, 50, 49, 52, 52, 52, 52, 52, 44, 48, 52, 52]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.093861777Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"downstairs","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.093893439Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"downstairs","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.09391894Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"downstairs","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.093946404Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"downstairs","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"Wait for a query_work_queue command to finish before sending IO Next flush: 94","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.093983418Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"downstairs","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.094010221Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"":"downstairs","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.094036924Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.094068426Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"client":"0","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.0940959Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"client":"1","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.094122693Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"client":"2","":"downstairs","session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.094149416Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.094173083Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"301b6b19-1e81-40d7-8d14-287274ddd923 is now active with session: 86588faa-b28e-40b5-b986-676d627e853e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.094201059Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.094225767Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408,"session_id":"86588faa-b28e-40b5-b986-676d627e853e"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.094250657Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408} {"msg":"Activated sub_volume 301b6b19-1e81-40d7-8d14-287274ddd923","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:11.094271197Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 533 len 40960 data: 7 8 6 8 6 6 6 6 7 8 02/30 Read block 379 len 8192 03/30 Flush 04/30 Flush 05/30 Write block 570 len 4096 data: 6 06/30 Write block 550 len 28672 data: 5 7 8 7 9 9 8 07/30 Write block 70 len 28672 data: 6 3 4 4 3 3 3 08/30 Read block 105 len 16384 09/30 Write block 180 len 4096 data: 6 10/30 Read block 26 len 32768 11/30 Write block 254 len 4096 data: 4 12/30 Read block 498 len 8192 13/30 Flush 14/30 Write block 226 len 24576 data: 4 4 5 5 6 6 15/30 Read block 453 len 24576 16/30 Read block 474 len 4096 17/30 Read block 112 len 16384 18/30 Read block 378 len 40960 19/30 Read block 172 len 28672 20/30 Write block 141 len 8192 data: 7 6 21/30 Write block 462 len 28672 data: 7 6 5 4 5 4 4 22/30 Write block 340 len 16384 data: 5 5 5 3 23/30 Read block 396 len 12288 24/30 Write block 436 len 28672 data: 8 11 12 8 7 6 6 25/30 Flush 26/30 Write block 246 len 40960 data: 7 7 6 9 8 4 5 4 5 5 27/30 Write block 429 len 28672 data: 7 7 6 5 6 6 6 28/30 Write block 157 len 20480 data: 3 2 3 4 6 29/30 Write block 283 len 16384 data: 5 6 5 6 30/30 Read block 31 len 16384 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 1 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 28 29 Done 29 30 2 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:11.126806563Z","hostname":"bmat-EVT22200007-0000ead9","pid":1408} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 93 93 93 F F F 1 020-039 50 50 50 88 88 88 F F F 2 040-059 49 49 49 86 86 86 F F F 3 060-079 54 54 52 96 96 90 F F F <--- 4 080-099 52 52 52 90 90 90 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 52 52 52 90 90 90 F F F 7 140-159 54 54 52 97 97 91 T T F <--- 8 160-179 44 44 44 79 79 79 T T F <--- 9 180-199 54 54 48 96 96 85 F F F <--- 10 200-219 52 52 52 90 90 90 F F F 11 220-239 54 54 52 97 97 92 F F F <--- 12 240-259 54 54 52 96 96 91 T T F <--- 13 260-279 50 50 50 87 87 87 F F F 14 280-299 50 50 50 88 88 88 T T F <--- 15 300-319 52 52 52 91 91 91 F F F 16 320-339 53 53 53 93 93 93 F F F 17 340-359 54 54 47 97 97 83 F F F <--- 18 360-379 46 46 46 82 82 82 F F F 19 380-399 50 50 50 87 87 87 F F F 20 400-419 53 53 53 93 93 93 F F F 21 420-439 54 54 43 97 97 76 T T F <--- 22 440-459 54 54 52 97 97 90 F F F <--- 23 460-479 54 54 52 97 97 90 F F F <--- 24 480-499 37 37 37 61 61 61 F F F 25 500-519 51 51 51 89 89 89 F F F 26 520-539 54 54 52 94 94 90 F F T <--- 27 540-559 54 54 51 96 96 89 F F T <--- 28 560-579 54 54 52 96 96 90 F F F <--- 29 580-599 53 53 53 93 93 93 F F F Max gen: 54, Max flush: 97 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 93 93 93 F F F 1 020-039 50 50 50 88 88 88 F F F 2 040-059 49 49 49 86 86 86 F F F 3 060-079 54 54 54 96 96 96 F F F 4 080-099 52 52 52 90 90 90 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 52 52 52 90 90 90 F F F 7 140-159 55 55 55 98 98 98 F F F 8 160-179 55 55 55 98 98 98 F F F 9 180-199 54 54 54 96 96 96 F F F 10 200-219 52 52 52 90 90 90 F F F 11 220-239 54 54 54 97 97 97 F F F 12 240-259 55 55 55 98 98 98 F F F 13 260-279 50 50 50 87 87 87 F F F 14 280-299 55 55 55 98 98 98 F F F 15 300-319 52 52 52 91 91 91 F F F 16 320-339 53 53 53 93 93 93 F F F 17 340-359 54 54 54 97 97 97 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 50 50 50 87 87 87 F F F 20 400-419 53 53 53 93 93 93 F F F 21 420-439 55 55 55 98 98 98 F F F 22 440-459 54 54 54 97 97 97 F F F 23 460-479 54 54 54 97 97 97 F F F 24 480-499 37 37 37 61 61 61 F F F 25 500-519 51 51 51 89 89 89 F F F 26 520-539 54 54 54 94 94 94 F F F 27 540-559 54 54 54 96 96 96 F F F 28 560-579 54 54 54 96 96 96 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 55, Max flush: 98 Begin loop 28 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.785303249Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.809488853Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.81004784Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.810129281Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.810152557Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.810185472Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416} {"msg":"Crucible 8dcef270-546b-4d60-9545-293930d651fb has session id: 14918cb8-b433-4814-bfa8-c75d3efce7a9","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.811106746Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"Upstairs opts: Upstairs UUID: 8dcef270-546b-4d60-9545-293930d651fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.811190693Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"Crucible stats registered with UUID: 8dcef270-546b-4d60-9545-293930d651fb","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.811222395Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:15.811248096Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.811695332Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.811773567Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.811799518Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.812119431Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.812158839Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416} {"msg":"8dcef270-546b-4d60-9545-293930d651fb active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.812254659Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.812281282Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"client":"0","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.812303566Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"client":"1","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.812324978Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"client":"2","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.81234623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.812389005Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.812489484Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:54004)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.812696364Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:39390)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.812733347Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:15.812905428Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:15.812945186Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:15.812974645Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.813264188Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.813293165Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.814878344Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"client":"0","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.81500256Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"client":"2","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.815764438Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:15.816100944Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.814812699Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:38320)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.815128504Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.817547564Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"client":"1","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"[0]R flush_numbers[0..12]: [93, 88, 86, 96, 90, 90, 90, 98, 98, 96, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.818867156Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"downstairs","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"[0]R generation[0..12]: [53, 50, 49, 54, 52, 52, 52, 55, 55, 54, 52, 54]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.818982705Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"downstairs","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.81914659Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"downstairs","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"[1]R flush_numbers[0..12]: [93, 88, 86, 96, 90, 90, 90, 98, 98, 96, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819220336Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"downstairs","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"[1]R generation[0..12]: [53, 50, 49, 54, 52, 52, 52, 55, 55, 54, 52, 54]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819251768Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"downstairs","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819284062Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"downstairs","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"[2]R flush_numbers[0..12]: [93, 88, 86, 96, 90, 90, 90, 98, 98, 96, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819313871Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"downstairs","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"[2]R generation[0..12]: [53, 50, 49, 54, 52, 52, 52, 55, 55, 54, 52, 54]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.81934355Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"downstairs","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819375393Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"downstairs","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:10:25.819401244Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"downstairs","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819435892Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"downstairs","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819461493Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"downstairs","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819487214Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"":"downstairs","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819514107Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819538315Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"client":"0","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.8195658Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"client":"1","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819600428Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"client":"2","":"downstairs","session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819628704Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819654626Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"8dcef270-546b-4d60-9545-293930d651fb is now active with session: 14918cb8-b433-4814-bfa8-c75d3efce7a9","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.81969215Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819716588Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416,"session_id":"14918cb8-b433-4814-bfa8-c75d3efce7a9"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819742489Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416} {"msg":"Activated sub_volume 8dcef270-546b-4d60-9545-293930d651fb","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:25.819764703Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 78 len 20480 data: 2 3 3 4 5 02/30 Flush 03/30 Write block 519 len 28672 data: 4 3 3 5 5 5 5 04/30 Read block 542 len 24576 05/30 Write block 391 len 40960 data: 5 6 4 5 4 4 5 5 5 7 06/30 Write block 564 len 36864 data: 7 5 5 4 5 5 7 6 7 07/30 Flush 08/30 Write block 185 len 28672 data: 4 5 7 8 7 6 8 09/30 Read block 464 len 20480 10/30 Read block 448 len 40960 11/30 Write block 229 len 32768 data: 6 7 7 7 8 8 6 6 12/30 Read block 138 len 40960 13/30 Write block 130 len 24576 data: 5 5 5 6 7 6 14/30 Read block 477 len 36864 15/30 Write block 286 len 8192 data: 7 6 16/30 Read block 135 len 32768 17/30 Write block 209 len 16384 data: 5 6 6 4 18/30 Write block 304 len 24576 data: 4 3 4 6 6 6 19/30 Read block 395 len 20480 20/30 Read block 330 len 16384 21/30 Read block 20 len 16384 22/30 Flush 23/30 Write block 452 len 36864 data: 4 4 6 5 7 10 11 10 10 24/30 Read block 237 len 36864 25/30 Write block 80 len 32768 data: 4 5 6 4 5 4 5 5 26/30 Write block 205 len 24576 data: 6 6 7 6 6 7 27/30 Write block 471 len 40960 data: 4 5 5 6 6 5 5 5 6 6 28/30 Read block 332 len 16384 29/30 Write block 241 len 12288 data: 7 6 5 30/30 Write block 51 len 12288 data: 6 5 5 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 7 Done Sent Done false 1003 Acked Read 6 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 29 2 33 Done 28 1 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:25.86492774Z","hostname":"bmat-EVT22200007-0000ead9","pid":1416} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 93 93 93 F F F 1 020-039 50 50 50 88 88 88 F F F 2 040-059 49 49 49 86 86 86 T F T <--- 3 060-079 56 54 56 99 96 99 F T F <--- 4 080-099 56 52 56 99 90 99 T T T <--- 5 100-119 52 52 52 90 90 90 F F F 6 120-139 56 52 56 101 90 101 F F F <--- 7 140-159 55 55 55 98 98 98 F F F 8 160-179 55 55 55 98 98 98 F F F 9 180-199 56 54 56 101 96 101 F F F <--- 10 200-219 56 52 56 101 90 101 T F T <--- 11 220-239 56 54 56 101 97 101 F F F <--- 12 240-259 55 55 55 98 98 98 T F T <--- 13 260-279 50 50 50 87 87 87 F F F 14 280-299 56 55 56 101 98 101 F F F <--- 15 300-319 56 52 56 101 91 101 F F F <--- 16 320-339 53 53 53 93 93 93 F F F 17 340-359 54 54 54 97 97 97 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 56 50 56 100 87 100 F F F <--- 20 400-419 56 53 56 100 93 100 F F F <--- 21 420-439 55 55 55 98 98 98 F F F 22 440-459 54 54 54 97 97 97 T F T <--- 23 460-479 54 54 54 97 97 97 T F T <--- 24 480-499 37 37 37 61 61 61 T F T <--- 25 500-519 56 51 56 100 89 100 F F F <--- 26 520-539 56 54 56 100 94 100 F F F <--- 27 540-559 54 54 54 96 96 96 F F F 28 560-579 56 54 56 100 96 100 F F F <--- 29 580-599 53 53 53 93 93 93 F F F Max gen: 56, Max flush: 101 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 93 93 93 F F F 1 020-039 50 50 50 88 88 88 F F F 2 040-059 57 57 57 102 102 102 F F F 3 060-079 56 56 56 99 99 99 F F F 4 080-099 57 57 57 102 102 102 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 56 56 56 101 101 101 F F F 7 140-159 55 55 55 98 98 98 F F F 8 160-179 55 55 55 98 98 98 F F F 9 180-199 56 56 56 101 101 101 F F F 10 200-219 57 57 57 102 102 102 F F F 11 220-239 56 56 56 101 101 101 F F F 12 240-259 57 57 57 102 102 102 F F F 13 260-279 50 50 50 87 87 87 F F F 14 280-299 56 56 56 101 101 101 F F F 15 300-319 56 56 56 101 101 101 F F F 16 320-339 53 53 53 93 93 93 F F F 17 340-359 54 54 54 97 97 97 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 56 56 56 100 100 100 F F F 20 400-419 56 56 56 100 100 100 F F F 21 420-439 55 55 55 98 98 98 F F F 22 440-459 57 57 57 102 102 102 F F F 23 460-479 57 57 57 102 102 102 F F F 24 480-499 57 57 57 102 102 102 F F F 25 500-519 56 56 56 100 100 100 F F F 26 520-539 56 56 56 100 100 100 F F F 27 540-559 54 54 54 96 96 96 F F F 28 560-579 56 56 56 100 100 100 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 57, Max flush: 102 Begin loop 29 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.551909475Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:30.577731489Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.577826507Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.600111398Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.600633371Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.600739101Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.600775663Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.600833538Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424} {"msg":"Crucible e21f2f2f-ce58-41a1-9ad7-762dc699ef12 has session id: 07d82f6f-3d03-422d-bb94-09c81edf72d6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.601960669Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"Upstairs opts: Upstairs UUID: e21f2f2f-ce58-41a1-9ad7-762dc699ef12, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.602055307Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"Crucible stats registered with UUID: e21f2f2f-ce58-41a1-9ad7-762dc699ef12","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.602101839Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:30.60214286Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.602520047Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.60259247Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.602638581Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.602860821Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.602901532Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424} {"msg":"e21f2f2f-ce58-41a1-9ad7-762dc699ef12 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.60308763Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.6031328Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"client":"0","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.603176947Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"client":"1","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.603220674Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"client":"2","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.60326399Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.603308878Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.603353797Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:64866)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.603855681Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:36531)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.603939347Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:49339)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.60397115Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.605454817Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"client":"1","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.605506329Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"client":"2","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.606190193Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"client":"0","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.606340911Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.606373326Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"[0]R flush_numbers[0..12]: [93, 88, 102, 99, 102, 90, 101, 98, 98, 101, 102, 101]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607237126Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"downstairs","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"[0]R generation[0..12]: [53, 50, 57, 56, 57, 52, 56, 55, 55, 56, 57, 56]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607274029Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"downstairs","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607303658Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"downstairs","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"[1]R flush_numbers[0..12]: [93, 88, 102, 99, 102, 90, 101, 98, 98, 101, 102, 101]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607331613Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"downstairs","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"[1]R generation[0..12]: [53, 50, 57, 56, 57, 52, 56, 55, 55, 56, 57, 56]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607359057Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"downstairs","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607388015Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"downstairs","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"[2]R flush_numbers[0..12]: [93, 88, 102, 99, 102, 90, 101, 98, 98, 101, 102, 101]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607415699Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"downstairs","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"[2]R generation[0..12]: [53, 50, 57, 56, 57, 52, 56, 55, 55, 56, 57, 56]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607450348Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"downstairs","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607479706Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"downstairs","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607504014Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"downstairs","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607528903Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"downstairs","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607552691Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"downstairs","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607575907Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"":"downstairs","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607600485Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.60762313Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"client":"0","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607648049Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"client":"1","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607672698Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"client":"2","":"downstairs","session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607698289Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607721655Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"e21f2f2f-ce58-41a1-9ad7-762dc699ef12 is now active with session: 07d82f6f-3d03-422d-bb94-09c81edf72d6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607748308Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607770291Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424,"session_id":"07d82f6f-3d03-422d-bb94-09c81edf72d6"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607793567Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424} {"msg":"Activated sub_volume e21f2f2f-ce58-41a1-9ad7-762dc699ef12","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:30.607813978Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 232 len 12288 data: 8 9 9 02/30 Read block 165 len 24576 03/30 Read block 22 len 20480 04/30 Flush 05/30 Write block 156 len 40960 data: 3 4 3 4 5 7 6 8 8 9 06/30 Write block 401 len 8192 data: 8 8 07/30 Read block 456 len 12288 08/30 Read block 593 len 24576 09/30 Write block 445 len 24576 data: 6 6 4 5 5 4 10/30 Write block 161 len 20480 data: 8 7 9 9 10 11/30 Read block 245 len 36864 12/30 Read block 289 len 32768 13/30 Read block 247 len 36864 14/30 Flush 15/30 Write block 255 len 4096 data: 6 16/30 Write block 577 len 8192 data: 5 6 17/30 Write block 272 len 36864 data: 6 5 5 3 4 4 4 5 5 18/30 Read block 494 len 36864 19/30 Read block 323 len 8192 20/30 Write block 302 len 24576 data: 3 3 5 4 5 7 21/30 Read block 108 len 20480 22/30 Write block 193 len 16384 data: 9 8 7 7 23/30 Flush 24/30 Write block 534 len 24576 data: 9 7 9 7 7 7 25/30 Write block 303 len 32768 data: 4 6 5 6 8 7 7 7 26/30 Read block 86 len 32768 27/30 Read block 165 len 40960 28/30 Write block 146 len 40960 data: 7 8 8 6 6 5 4 4 5 3 29/30 Read block 212 len 8192 30/30 Write block 510 len 8192 data: 7 6 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Sent Done Done false 1001 Acked Read 6 Sent Done Done false 1002 Acked Read 5 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Read 8 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:30.643596532Z","hostname":"bmat-EVT22200007-0000ead9","pid":1424} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 93 93 93 F F F 1 020-039 50 50 50 88 88 88 F F F 2 040-059 57 57 57 102 102 102 F F F 3 060-079 56 56 56 99 99 99 F F F 4 080-099 57 57 57 102 102 102 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 56 56 56 101 101 101 F F F 7 140-159 55 58 58 98 104 104 F T T <--- 8 160-179 55 58 58 98 104 104 F F F <--- 9 180-199 56 58 58 101 105 105 F F F <--- 10 200-219 57 57 57 102 102 102 F F F 11 220-239 56 58 58 101 103 103 F F F <--- 12 240-259 57 58 58 102 105 105 F F F <--- 13 260-279 50 58 58 87 105 105 F F F <--- 14 280-299 56 58 58 101 105 105 F F F <--- 15 300-319 56 58 58 101 105 105 F T T <--- 16 320-339 53 53 53 93 93 93 F F F 17 340-359 54 54 54 97 97 97 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 56 56 56 100 100 100 F F F 20 400-419 56 58 58 100 104 104 F F F <--- 21 420-439 55 55 55 98 98 98 F F F 22 440-459 57 58 58 102 104 104 F F F <--- 23 460-479 57 57 57 102 102 102 F F F 24 480-499 57 57 57 102 102 102 F F F 25 500-519 56 56 56 100 100 100 F T T <--- 26 520-539 56 56 56 100 100 100 F T T <--- 27 540-559 54 54 54 96 96 96 F F F 28 560-579 56 58 58 100 105 105 F F F <--- 29 580-599 53 53 53 93 93 93 F F F Max gen: 58, Max flush: 105 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 53 53 53 93 93 93 F F F 1 020-039 50 50 50 88 88 88 F F F 2 040-059 57 57 57 102 102 102 F F F 3 060-079 56 56 56 99 99 99 F F F 4 080-099 57 57 57 102 102 102 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 56 56 56 101 101 101 F F F 7 140-159 59 59 59 106 106 106 F F F 8 160-179 58 58 58 104 104 104 F F F 9 180-199 58 58 58 105 105 105 F F F 10 200-219 57 57 57 102 102 102 F F F 11 220-239 58 58 58 103 103 103 F F F 12 240-259 58 58 58 105 105 105 F F F 13 260-279 58 58 58 105 105 105 F F F 14 280-299 58 58 58 105 105 105 F F F 15 300-319 59 59 59 106 106 106 F F F 16 320-339 53 53 53 93 93 93 F F F 17 340-359 54 54 54 97 97 97 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 56 56 56 100 100 100 F F F 20 400-419 58 58 58 104 104 104 F F F 21 420-439 55 55 55 98 98 98 F F F 22 440-459 58 58 58 104 104 104 F F F 23 460-479 57 57 57 102 102 102 F F F 24 480-499 57 57 57 102 102 102 F F F 25 500-519 59 59 59 106 106 106 F F F 26 520-539 59 59 59 106 106 106 F F F 27 540-559 54 54 54 96 96 96 F F F 28 560-579 58 58 58 105 105 105 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 59, Max flush: 106 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.31168965Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.337092314Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.337924561Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.338256508Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.338295856Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.338332158Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432} {"msg":"Crucible 07ec18d8-ea4f-4ade-9bb1-5f86e4226d6c has session id: 3b3203a8-93fa-42f0-96ba-0646de520cdc","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.339475672Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"Upstairs opts: Upstairs UUID: 07ec18d8-ea4f-4ade-9bb1-5f86e4226d6c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.339545641Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"Crucible stats registered with UUID: 07ec18d8-ea4f-4ade-9bb1-5f86e4226d6c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.339575971Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:35.339601572Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.34026247Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.340397628Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.340443719Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.340492976Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.34061079Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432} {"msg":"07ec18d8-ea4f-4ade-9bb1-5f86e4226d6c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.340662823Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.340701169Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"client":"0","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.34074232Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"client":"1","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.34078293Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"client":"2","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.34082306Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.340866065Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.340907257Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"ds_connection connected from Ok(127.0.0.1:36852)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.341082564Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"ds_connection connected from Ok(127.0.0.1:57910)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.34113602Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"ds_connection connected from Ok(127.0.0.1:51769)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.341179596Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.343784283Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"client":"0","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.343877046Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"client":"1","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.344107763Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"client":"2","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.344556772Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.344683463Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"[0]R flush_numbers[0..12]: [93, 88, 102, 99, 102, 90, 101, 106, 104, 105, 102, 103]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.344993437Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"downstairs","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"[0]R generation[0..12]: [53, 50, 57, 56, 57, 52, 56, 59, 58, 58, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.34504038Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"downstairs","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345073535Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"downstairs","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"[1]R flush_numbers[0..12]: [93, 88, 102, 99, 102, 90, 101, 106, 104, 105, 102, 103]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345104246Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"downstairs","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"[1]R generation[0..12]: [53, 50, 57, 56, 57, 52, 56, 59, 58, 58, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345134105Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"downstairs","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345165788Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"downstairs","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"[2]R flush_numbers[0..12]: [93, 88, 102, 99, 102, 90, 101, 106, 104, 105, 102, 103]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345196268Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"downstairs","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"[2]R generation[0..12]: [53, 50, 57, 56, 57, 52, 56, 59, 58, 58, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345226177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"downstairs","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345258051Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"downstairs","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345284443Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"downstairs","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345311627Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"downstairs","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-04T01:10:35.345337127Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"downstairs","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345376285Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"":"downstairs","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345403138Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345428097Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"client":"0","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345459089Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"client":"1","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345496322Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"client":"2","":"downstairs","session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345524959Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.34555094Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"07ec18d8-ea4f-4ade-9bb1-5f86e4226d6c is now active with session: 3b3203a8-93fa-42f0-96ba-0646de520cdc","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345580098Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345604446Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432,"session_id":"3b3203a8-93fa-42f0-96ba-0646de520cdc"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345630017Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432} {"msg":"Activated sub_volume 07ec18d8-ea4f-4ade-9bb1-5f86e4226d6c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:35.345652051Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 89 len 28672 data: 5 4 5 5 6 5 4 02/30 Read block 421 len 20480 03/30 Write block 31 len 36864 data: 4 5 4 4 5 6 6 5 3 04/30 Write block 433 len 16384 data: 7 7 7 9 05/30 Read block 152 len 4096 06/30 Write block 19 len 24576 data: 4 5 6 7 5 5 07/30 Write block 284 len 4096 data: 7 08/30 Flush 09/30 Write block 9 len 20480 data: 6 3 3 3 4 10/30 Write block 139 len 24576 data: 9 8 8 7 7 6 11/30 Write block 222 len 4096 data: 2 12/30 Write block 147 len 8192 data: 9 9 13/30 Flush 14/30 Flush 15/30 Write block 543 len 4096 data: 7 16/30 Read block 573 len 16384 17/30 Write block 550 len 24576 data: 6 8 9 8 10 10 18/30 Write block 223 len 36864 data: 2 3 3 5 5 6 7 8 8 19/30 Flush 20/30 Write block 462 len 40960 data: 8 7 6 5 6 5 5 5 5 5 21/30 Write block 156 len 32768 data: 4 5 4 5 6 9 8 10 22/30 Write block 5 len 28672 data: 9 10 10 6 7 4 4 23/30 Read block 592 len 8192 24/30 Read block 370 len 12288 25/30 Flush 26/30 Write block 39 len 32768 data: 4 3 3 2 2 2 4 3 27/30 Read block 259 len 12288 28/30 Read block 241 len 12288 29/30 Write block 328 len 4096 data: 4 30/30 Write block 88 len 8192 data: 5 6 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Sent false 1001 Acked Read 5 Done Done Sent false 1002 Acked Write 9 Done Done Sent false 1003 Acked Write 4 Done Done Sent false 1004 Acked Read 1 Done Done Sent false 1005 Acked Write 6 Done Done Sent false 1006 Acked Write 1 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 5 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Read 2 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Read 3 Sent Done Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 30 35 Done 27 28 0 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:35.375884387Z","hostname":"bmat-EVT22200007-0000ead9","pid":1432} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 53 111 111 93 F F F <--- 1 020-039 60 60 50 107 107 88 T T F <--- 2 040-059 57 57 57 102 102 102 T T F <--- 3 060-079 56 56 56 99 99 99 F F F 4 080-099 60 60 57 107 107 102 T T F <--- 5 100-119 52 52 52 90 90 90 F F F 6 120-139 60 60 56 108 108 101 F F F <--- 7 140-159 60 60 59 111 111 106 F F F <--- 8 160-179 60 60 58 111 111 104 F F F <--- 9 180-199 58 58 58 105 105 105 F F F 10 200-219 57 57 57 102 102 102 F F F 11 220-239 60 60 58 110 110 103 F F F <--- 12 240-259 58 58 58 105 105 105 F F F 13 260-279 58 58 58 105 105 105 F F F 14 280-299 60 60 58 107 107 105 F F F <--- 15 300-319 59 59 59 106 106 106 F F F 16 320-339 53 53 53 93 93 93 T T F <--- 17 340-359 54 54 54 97 97 97 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 56 56 56 100 100 100 F F F 20 400-419 58 58 58 104 104 104 F F F 21 420-439 60 60 55 107 107 98 F F F <--- 22 440-459 58 58 58 104 104 104 F F F 23 460-479 60 60 57 111 111 102 F F F <--- 24 480-499 57 57 57 102 102 102 F F F 25 500-519 59 59 59 106 106 106 F F F 26 520-539 59 59 59 106 106 106 F F F 27 540-559 60 60 54 110 110 96 F F F <--- 28 560-579 58 58 58 105 105 105 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 60, Max flush: 111 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 111 111 111 F F F 1 020-039 61 61 61 112 112 112 F F F 2 040-059 61 61 61 112 112 112 F F F 3 060-079 56 56 56 99 99 99 F F F 4 080-099 61 61 61 112 112 112 F F F 5 100-119 52 52 52 90 90 90 F F F 6 120-139 60 60 60 108 108 108 F F F 7 140-159 60 60 60 111 111 111 F F F 8 160-179 60 60 60 111 111 111 F F F 9 180-199 58 58 58 105 105 105 F F F 10 200-219 57 57 57 102 102 102 F F F 11 220-239 60 60 60 110 110 110 F F F 12 240-259 58 58 58 105 105 105 F F F 13 260-279 58 58 58 105 105 105 F F F 14 280-299 60 60 60 107 107 107 F F F 15 300-319 59 59 59 106 106 106 F F F 16 320-339 61 61 61 112 112 112 F F F 17 340-359 54 54 54 97 97 97 F F F 18 360-379 46 46 46 82 82 82 F F F 19 380-399 56 56 56 100 100 100 F F F 20 400-419 58 58 58 104 104 104 F F F 21 420-439 60 60 60 107 107 107 F F F 22 440-459 58 58 58 104 104 104 F F F 23 460-479 60 60 60 111 111 111 F F F 24 480-499 57 57 57 102 102 102 F F F 25 500-519 59 59 59 106 106 106 F F F 26 520-539 59 59 59 106 106 106 F F F 27 540-559 60 60 60 110 110 110 F F F 28 560-579 58 58 58 105 105 105 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 61, Max flush: 112 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.025807078Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.053047029Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.053761253Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.053853746Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.053885329Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.053934857Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440} {"msg":"Crucible ad82869d-07d0-4199-8903-1eee991b7367 has session id: 8371b2d2-1fec-479a-8e48-485918998524","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.055011599Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"Upstairs opts: Upstairs UUID: ad82869d-07d0-4199-8903-1eee991b7367, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.055100345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"Crucible stats registered with UUID: ad82869d-07d0-4199-8903-1eee991b7367","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.055140905Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:40.055175904Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.055651726Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.055715432Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.055754199Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.056083361Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.056121727Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440} {"msg":"ad82869d-07d0-4199-8903-1eee991b7367 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.056493423Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.056530125Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"client":"0","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.056567509Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"client":"1","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.056604232Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"client":"2","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.056644081Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.056682347Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.056720091Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:40.057163509Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"ds_connection connected from Ok(127.0.0.1:42169)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.057256634Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"ds_connection connected from Ok(127.0.0.1:55421)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.057288126Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:40.057341782Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:40.057377493Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.057402432Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.057430468Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.059075705Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"client":"1","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.059123039Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"client":"0","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.06017427Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:40.060248136Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.058374732Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"ds_connection connected from Ok(127.0.0.1:49753)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.058474609Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.060751658Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"client":"2","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"[0]R flush_numbers[0..12]: [111, 112, 112, 99, 112, 90, 108, 111, 111, 105, 102, 110]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062151629Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"downstairs","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"[0]R generation[0..12]: [60, 61, 61, 56, 61, 52, 60, 60, 60, 58, 57, 60]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062251177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"downstairs","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062285965Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"downstairs","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"[1]R flush_numbers[0..12]: [111, 112, 112, 99, 112, 90, 108, 111, 111, 105, 102, 110]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062317127Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"downstairs","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"[1]R generation[0..12]: [60, 61, 61, 56, 61, 52, 60, 60, 60, 58, 57, 60]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062347006Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"downstairs","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.06237927Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"downstairs","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"[2]R flush_numbers[0..12]: [111, 112, 112, 99, 112, 90, 108, 111, 111, 105, 102, 110]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062410452Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"downstairs","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"[2]R generation[0..12]: [60, 61, 61, 56, 61, 52, 60, 60, 60, 58, 57, 60]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062440592Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"downstairs","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062472334Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"downstairs","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062498176Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"downstairs","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062525019Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"downstairs","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062550509Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"downstairs","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.06257622Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"":"downstairs","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062603173Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062627411Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"client":"0","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062654545Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"client":"1","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062685086Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"client":"2","":"downstairs","session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.06271269Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:10:50.062738421Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"ad82869d-07d0-4199-8903-1eee991b7367 is now active with session: 8371b2d2-1fec-479a-8e48-485918998524","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062775815Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062800073Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440,"session_id":"8371b2d2-1fec-479a-8e48-485918998524"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062825654Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440} {"msg":"Activated sub_volume ad82869d-07d0-4199-8903-1eee991b7367","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:50.062847718Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 220 len 28672 data: 2 2 3 3 4 4 6 02/30 Read block 500 len 4096 03/30 Write block 264 len 24576 data: 8 6 6 5 7 7 04/30 Write block 43 len 20480 data: 3 3 5 4 5 05/30 Write block 357 len 12288 data: 7 9 9 06/30 Write block 453 len 24576 data: 5 7 6 8 11 12 07/30 Read block 509 len 8192 08/30 Flush 09/30 Write block 528 len 40960 data: 5 6 5 5 5 8 10 8 10 8 10/30 Write block 179 len 40960 data: 4 7 6 6 6 5 5 6 8 9 11/30 Read block 182 len 4096 12/30 Read block 440 len 8192 13/30 Read block 304 len 40960 14/30 Flush 15/30 Write block 136 len 8192 data: 7 7 16/30 Read block 333 len 20480 17/30 Read block 363 len 36864 18/30 Write block 216 len 16384 data: 4 5 5 2 19/30 Read block 52 len 32768 20/30 Write block 159 len 20480 data: 6 7 10 9 11 21/30 Write block 117 len 4096 data: 8 22/30 Read block 0 len 20480 23/30 Flush 24/30 Write block 222 len 20480 data: 4 4 5 5 7 25/30 Read block 170 len 20480 26/30 Write block 336 len 24576 data: 6 6 4 4 6 6 27/30 Read block 203 len 8192 28/30 Write block 385 len 24576 data: 5 4 4 4 4 4 29/30 Read block 229 len 20480 30/30 Write block 376 len 40960 data: 3 3 3 4 3 5 5 4 4 6 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Sent false 1001 Acked Read 1 Done Done Sent false 1002 Acked Write 6 Done Done Sent false 1003 Acked Write 5 Done Done Sent false 1004 Acked Write 3 Done Done Sent false 1005 Acked Write 6 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:50.101485696Z","hostname":"bmat-EVT22200007-0000ead9","pid":1440} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 111 111 111 F F F 1 020-039 61 61 61 112 112 112 F F F 2 040-059 62 62 61 113 113 112 F F F <--- 3 060-079 56 56 56 99 99 99 F F F 4 080-099 61 61 61 112 112 112 F F F 5 100-119 62 62 52 115 115 90 F F F <--- 6 120-139 62 62 60 115 115 108 F F F <--- 7 140-159 62 62 60 115 115 111 F F F <--- 8 160-179 62 62 60 115 115 111 F F F <--- 9 180-199 62 62 58 114 114 105 F F F <--- 10 200-219 62 62 57 115 115 102 F F F <--- 11 220-239 62 62 60 113 113 110 T T F <--- 12 240-259 58 58 58 105 105 105 F F F 13 260-279 62 62 58 113 113 105 F F F <--- 14 280-299 60 60 60 107 107 107 F F F 15 300-319 59 59 59 106 106 106 F F F 16 320-339 61 61 61 112 112 112 T T F <--- 17 340-359 62 62 54 113 113 97 T T F <--- 18 360-379 46 46 46 82 82 82 T T F <--- 19 380-399 56 56 56 100 100 100 T T F <--- 20 400-419 58 58 58 104 104 104 F F F 21 420-439 60 60 60 107 107 107 F F F 22 440-459 62 62 58 113 113 104 F F F <--- 23 460-479 60 60 60 111 111 111 F F F 24 480-499 57 57 57 102 102 102 F F F 25 500-519 59 59 59 106 106 106 F F F 26 520-539 62 62 59 114 114 106 F F F <--- 27 540-559 60 60 60 110 110 110 F F F 28 560-579 58 58 58 105 105 105 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 62, Max flush: 115 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 111 111 111 F F F 1 020-039 61 61 61 112 112 112 F F F 2 040-059 62 62 62 113 113 113 F F F 3 060-079 56 56 56 99 99 99 F F F 4 080-099 61 61 61 112 112 112 F F F 5 100-119 62 62 62 115 115 115 F F F 6 120-139 62 62 62 115 115 115 F F F 7 140-159 62 62 62 115 115 115 F F F 8 160-179 62 62 62 115 115 115 F F F 9 180-199 62 62 62 114 114 114 F F F 10 200-219 62 62 62 115 115 115 F F F 11 220-239 63 63 63 116 116 116 F F F 12 240-259 58 58 58 105 105 105 F F F 13 260-279 62 62 62 113 113 113 F F F 14 280-299 60 60 60 107 107 107 F F F 15 300-319 59 59 59 106 106 106 F F F 16 320-339 63 63 63 116 116 116 F F F 17 340-359 63 63 63 116 116 116 F F F 18 360-379 63 63 63 116 116 116 F F F 19 380-399 63 63 63 116 116 116 F F F 20 400-419 58 58 58 104 104 104 F F F 21 420-439 60 60 60 107 107 107 F F F 22 440-459 62 62 62 113 113 113 F F F 23 460-479 60 60 60 111 111 111 F F F 24 480-499 57 57 57 102 102 102 F F F 25 500-519 59 59 59 106 106 106 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 60 60 60 110 110 110 F F F 28 560-579 58 58 58 105 105 105 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 63, Max flush: 116 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.816759623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.845360629Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.845937962Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.846010606Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.846034283Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.846069643Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448} {"msg":"Crucible 7f24fc28-7875-4543-bcb6-a9ab942eabb7 has session id: a33e2a08-e19a-41a6-8018-b607164c9009","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.847309798Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"Upstairs opts: Upstairs UUID: 7f24fc28-7875-4543-bcb6-a9ab942eabb7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.847370378Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"Crucible stats registered with UUID: 7f24fc28-7875-4543-bcb6-a9ab942eabb7","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.847405798Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:54.847431529Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.847916339Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.847970726Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.847999774Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.848308355Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.848340128Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448} {"msg":"7f24fc28-7875-4543-bcb6-a9ab942eabb7 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.848491377Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.848520665Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"client":"0","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.848547899Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"client":"1","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.848574592Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"client":"2","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.848608309Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.848682005Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.848712866Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"ds_connection connected from Ok(127.0.0.1:37133)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.849243516Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"ds_connection connected from Ok(127.0.0.1:61806)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.849322142Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"ds_connection connected from Ok(127.0.0.1:47556)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.849360969Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.850919274Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"client":"1","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.851089421Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"client":"0","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.851128438Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"client":"2","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.851662756Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.851978662Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"[0]R flush_numbers[0..12]: [111, 112, 113, 99, 112, 115, 115, 115, 115, 114, 115, 116]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.852593138Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"downstairs","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"[0]R generation[0..12]: [60, 61, 62, 56, 61, 62, 62, 62, 62, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.852629931Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"downstairs","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.852661664Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"downstairs","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"[1]R flush_numbers[0..12]: [111, 112, 113, 99, 112, 115, 115, 115, 115, 114, 115, 116]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.852692194Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"downstairs","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"[1]R generation[0..12]: [60, 61, 62, 56, 61, 62, 62, 62, 62, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.85272032Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"downstairs","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.852751452Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"downstairs","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"[2]R flush_numbers[0..12]: [111, 112, 113, 99, 112, 115, 115, 115, 115, 114, 115, 116]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.852781702Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"downstairs","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"[2]R generation[0..12]: [60, 61, 62, 56, 61, 62, 62, 62, 62, 62, 62, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.852810539Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"downstairs","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.852847512Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"downstairs","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.852874395Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"downstairs","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.852900687Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"downstairs","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.852926107Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"downstairs","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.852950917Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"":"downstairs","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.852977088Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.853001386Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"client":"0","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.853027278Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"client":"1","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.85305365Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"client":"2","":"downstairs","session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.853080092Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.853104992Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"7f24fc28-7875-4543-bcb6-a9ab942eabb7 is now active with session: a33e2a08-e19a-41a6-8018-b607164c9009","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.853133759Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.853158047Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448,"session_id":"a33e2a08-e19a-41a6-8018-b607164c9009"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.853183657Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448} {"msg":"Activated sub_volume 7f24fc28-7875-4543-bcb6-a9ab942eabb7","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:54.853204929Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 234 len 32768 data: 10 7 7 7 7 9 8 8 02/30 Flush 03/30 Flush 04/30 Write block 74 len 20480 data: 4 4 4 3 3 05/30 Read block 249 len 40960 06/30 Write block 512 len 24576 data: 6 6 7 6 5 5 07/30 Write block 378 len 20480 data: 4 5 4 6 6 08/30 Read block 197 len 32768 09/30 Read block 241 len 12288 10/30 Read block 342 len 40960 11/30 Write block 126 len 20480 data: 5 6 5 6 6 12/30 Flush 13/30 Write block 212 len 4096 data: 5 14/30 Read block 478 len 32768 15/30 Write block 151 len 12288 data: 6 5 5 16/30 Write block 143 len 24576 data: 8 7 6 8 10 10 17/30 Write block 109 len 24576 data: 3 3 4 5 10 9 18/30 Write block 315 len 24576 data: 5 7 6 5 5 8 19/30 Write block 475 len 24576 data: 7 6 6 6 7 7 20/30 Read block 205 len 8192 21/30 Write block 217 len 20480 data: 6 6 3 3 3 22/30 Read block 516 len 16384 23/30 Flush 24/30 Write block 459 len 4096 data: 11 25/30 Flush 26/30 Write block 268 len 36864 data: 8 8 6 6 7 6 6 4 5 27/30 Read block 142 len 40960 28/30 Write block 278 len 16384 data: 5 6 6 6 29/30 Read block 218 len 24576 30/30 Read block 104 len 28672 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Sent false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 5 Done Done Sent false 1004 Acked Read 10 Done Done Sent false 1005 Acked Write 6 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:54.888550729Z","hostname":"bmat-EVT22200007-0000ead9","pid":1448} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 111 111 111 F F F 1 020-039 61 61 61 112 112 112 F F F 2 040-059 62 62 62 113 113 113 F F F 3 060-079 64 64 56 119 119 99 F F F <--- 4 080-099 61 61 61 112 112 112 F F F 5 100-119 64 64 62 120 120 115 F F F <--- 6 120-139 64 64 62 119 119 115 F F F <--- 7 140-159 64 64 62 120 120 115 F F F <--- 8 160-179 62 62 62 115 115 115 F F F 9 180-199 62 62 62 114 114 114 F F F 10 200-219 64 64 62 120 120 115 F F F <--- 11 220-239 64 64 63 120 120 116 F F F <--- 12 240-259 64 64 58 117 117 105 F F F <--- 13 260-279 62 62 62 113 113 113 T T F <--- 14 280-299 60 60 60 107 107 107 T T F <--- 15 300-319 64 64 59 120 120 106 F F F <--- 16 320-339 64 64 63 120 120 116 F F F <--- 17 340-359 63 63 63 116 116 116 F F F 18 360-379 64 64 63 119 119 116 F F F <--- 19 380-399 64 64 63 119 119 116 F F F <--- 20 400-419 58 58 58 104 104 104 F F F 21 420-439 60 60 60 107 107 107 F F F 22 440-459 64 64 62 121 121 113 F F F <--- 23 460-479 64 64 60 120 120 111 F F F <--- 24 480-499 64 64 57 120 120 102 F F F <--- 25 500-519 64 64 59 119 119 106 F F F <--- 26 520-539 62 62 62 114 114 114 F F F 27 540-559 60 60 60 110 110 110 F F F 28 560-579 58 58 58 105 105 105 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 64, Max flush: 121 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 111 111 111 F F F 1 020-039 61 61 61 112 112 112 F F F 2 040-059 62 62 62 113 113 113 F F F 3 060-079 64 64 64 119 119 119 F F F 4 080-099 61 61 61 112 112 112 F F F 5 100-119 64 64 64 120 120 120 F F F 6 120-139 64 64 64 119 119 119 F F F 7 140-159 64 64 64 120 120 120 F F F 8 160-179 62 62 62 115 115 115 F F F 9 180-199 62 62 62 114 114 114 F F F 10 200-219 64 64 64 120 120 120 F F F 11 220-239 64 64 64 120 120 120 F F F 12 240-259 64 64 64 117 117 117 F F F 13 260-279 65 65 65 122 122 122 F F F 14 280-299 65 65 65 122 122 122 F F F 15 300-319 64 64 64 120 120 120 F F F 16 320-339 64 64 64 120 120 120 F F F 17 340-359 63 63 63 116 116 116 F F F 18 360-379 64 64 64 119 119 119 F F F 19 380-399 64 64 64 119 119 119 F F F 20 400-419 58 58 58 104 104 104 F F F 21 420-439 60 60 60 107 107 107 F F F 22 440-459 64 64 64 121 121 121 F F F 23 460-479 64 64 64 120 120 120 F F F 24 480-499 64 64 64 120 120 120 F F F 25 500-519 64 64 64 119 119 119 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 60 60 60 110 110 110 F F F 28 560-579 58 58 58 105 105 105 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 65, Max flush: 122 Begin loop 33 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.550411165Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.577383136Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.57827409Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.578379629Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.578419658Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.578479036Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456} {"msg":"Crucible 71094cfe-7c7a-4938-930a-0d118c5d4a6f has session id: c4bd3436-b2cd-459c-8056-7701c35bbfea","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.579478615Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"Upstairs opts: Upstairs UUID: 71094cfe-7c7a-4938-930a-0d118c5d4a6f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.579565968Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"Crucible stats registered with UUID: 71094cfe-7c7a-4938-930a-0d118c5d4a6f","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.579617199Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:59.579660906Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.580257417Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.580326734Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.580360661Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.580794901Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.580830331Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456} {"msg":"71094cfe-7c7a-4938-930a-0d118c5d4a6f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.581087951Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.581127109Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"client":"0","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.581157689Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"client":"1","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.581186727Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"client":"2","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.581239311Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.581347505Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.581790362Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"ds_connection connected from Ok(127.0.0.1:57505)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.582148741Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"ds_connection connected from Ok(127.0.0.1:52328)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.582251495Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"ds_connection connected from Ok(127.0.0.1:42542)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.582300702Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.583992712Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"client":"1","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.58414795Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"client":"0","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.58418839Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"client":"2","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.584937433Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585103221Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"[0]R flush_numbers[0..12]: [111, 112, 113, 119, 112, 120, 119, 120, 115, 114, 120, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585379778Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"downstairs","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"[0]R generation[0..12]: [60, 61, 62, 64, 61, 64, 64, 64, 62, 62, 64, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585423886Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"downstairs","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585456941Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"downstairs","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"[1]R flush_numbers[0..12]: [111, 112, 113, 119, 112, 120, 119, 120, 115, 114, 120, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585487852Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"downstairs","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"[1]R generation[0..12]: [60, 61, 62, 64, 61, 64, 64, 64, 62, 62, 64, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585517491Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"downstairs","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585549174Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"downstairs","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"[2]R flush_numbers[0..12]: [111, 112, 113, 119, 112, 120, 119, 120, 115, 114, 120, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585580556Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"downstairs","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"[2]R generation[0..12]: [60, 61, 62, 64, 61, 64, 64, 64, 62, 62, 64, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585610154Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"downstairs","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585641967Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"downstairs","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585668079Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"downstairs","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585704952Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"downstairs","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585734761Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"downstairs","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585760803Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"":"downstairs","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585787886Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:10:59.585812685Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"client":"0","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585859769Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"client":"1","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.58589063Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"client":"2","":"downstairs","session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585918545Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585944426Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"71094cfe-7c7a-4938-930a-0d118c5d4a6f is now active with session: c4bd3436-b2cd-459c-8056-7701c35bbfea","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585973554Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.585997892Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456,"session_id":"c4bd3436-b2cd-459c-8056-7701c35bbfea"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.586024024Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456} {"msg":"Activated sub_volume 71094cfe-7c7a-4938-930a-0d118c5d4a6f","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:10:59.586046168Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 471 len 16384 data: 6 6 6 7 02/30 Read block 573 len 16384 03/30 Flush 04/30 Write block 196 len 36864 data: 8 7 7 7 8 8 7 7 5 05/30 Write block 555 len 20480 data: 11 9 7 6 6 06/30 Flush 07/30 Write block 273 len 16384 data: 7 7 5 6 08/30 Read block 161 len 12288 09/30 Write block 144 len 24576 data: 8 7 9 11 11 7 10/30 Read block 189 len 8192 11/30 Read block 125 len 4096 12/30 Read block 337 len 8192 13/30 Write block 320 len 20480 data: 9 8 9 8 10 14/30 Write block 308 len 16384 data: 8 8 8 8 15/30 Read block 69 len 4096 16/30 Read block 281 len 36864 17/30 Read block 249 len 20480 18/30 Write block 365 len 40960 data: 6 4 4 4 4 4 4 4 4 4 19/30 Read block 546 len 20480 20/30 Write block 168 len 40960 data: 10 6 5 5 3 3 4 4 6 6 21/30 Read block 275 len 40960 22/30 Write block 52 len 20480 data: 6 6 5 4 5 23/30 Flush 24/30 Write block 116 len 24576 data: 9 9 8 7 8 5 25/30 Flush 26/30 Flush 27/30 Write block 72 len 28672 data: 5 5 5 5 5 4 4 28/30 Write block 275 len 20480 data: 6 7 5 6 7 29/30 Read block 591 len 32768 30/30 Write block 93 len 16384 data: 7 6 5 5 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 4 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 4 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Write 4 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:10:59.624266859Z","hostname":"bmat-EVT22200007-0000ead9","pid":1456} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 111 111 111 F F F 1 020-039 61 61 61 112 112 112 F F F 2 040-059 66 66 62 125 125 113 F F F <--- 3 060-079 64 64 64 119 119 119 T T F <--- 4 080-099 61 61 61 112 112 112 T T F <--- 5 100-119 66 66 64 126 126 120 F F F <--- 6 120-139 66 66 64 126 126 119 F F F <--- 7 140-159 66 66 64 125 125 120 F F F <--- 8 160-179 66 66 62 125 125 115 F F F <--- 9 180-199 66 66 62 124 124 114 F F F <--- 10 200-219 66 66 64 124 124 120 F F F <--- 11 220-239 64 64 64 120 120 120 F F F 12 240-259 64 64 64 117 117 117 F F F 13 260-279 66 66 65 125 125 122 T T F <--- 14 280-299 65 65 65 122 122 122 F F F 15 300-319 66 66 64 125 125 120 F F F <--- 16 320-339 66 66 64 125 125 120 F F F <--- 17 340-359 63 63 63 116 116 116 F F F 18 360-379 66 66 64 125 125 119 F F F <--- 19 380-399 64 64 64 119 119 119 F F F 20 400-419 58 58 58 104 104 104 F F F 21 420-439 60 60 60 107 107 107 F F F 22 440-459 64 64 64 121 121 121 F F F 23 460-479 66 66 64 123 123 120 F F T <--- 24 480-499 64 64 64 120 120 120 F F F 25 500-519 64 64 64 119 119 119 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 66 66 60 124 124 110 F F F <--- 28 560-579 58 58 58 105 105 105 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 66, Max flush: 126 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 111 111 111 F F F 1 020-039 61 61 61 112 112 112 F F F 2 040-059 66 66 66 125 125 125 F F F 3 060-079 67 67 67 127 127 127 F F F 4 080-099 67 67 67 127 127 127 F F F 5 100-119 66 66 66 126 126 126 F F F 6 120-139 66 66 66 126 126 126 F F F 7 140-159 66 66 66 125 125 125 F F F 8 160-179 66 66 66 125 125 125 F F F 9 180-199 66 66 66 124 124 124 F F F 10 200-219 66 66 66 124 124 124 F F F 11 220-239 64 64 64 120 120 120 F F F 12 240-259 64 64 64 117 117 117 F F F 13 260-279 67 67 67 127 127 127 F F F 14 280-299 65 65 65 122 122 122 F F F 15 300-319 66 66 66 125 125 125 F F F 16 320-339 66 66 66 125 125 125 F F F 17 340-359 63 63 63 116 116 116 F F F 18 360-379 66 66 66 125 125 125 F F F 19 380-399 64 64 64 119 119 119 F F F 20 400-419 58 58 58 104 104 104 F F F 21 420-439 60 60 60 107 107 107 F F F 22 440-459 64 64 64 121 121 121 F F F 23 460-479 66 66 66 123 123 123 F F F 24 480-499 64 64 64 120 120 120 F F F 25 500-519 64 64 64 119 119 119 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 66 66 66 124 124 124 F F F 28 560-579 58 58 58 105 105 105 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 67, Max flush: 127 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.32112142Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.343972302Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.344602189Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.344679863Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.344709261Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.344748589Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464} {"msg":"Crucible fd0eda8d-bc68-4d66-9a16-ca2fef446f0c has session id: dd04104d-b4ab-4dfc-bd5b-2905aa60dd36","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.346028863Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"Upstairs opts: Upstairs UUID: fd0eda8d-bc68-4d66-9a16-ca2fef446f0c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.346168119Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"Crucible stats registered with UUID: fd0eda8d-bc68-4d66-9a16-ca2fef446f0c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.346237176Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:04.346288648Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.346474125Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.346548552Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.346581668Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.346935158Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.346964586Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464} {"msg":"fd0eda8d-bc68-4d66-9a16-ca2fef446f0c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.347172277Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.347202808Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"client":"0","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.347230863Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"client":"1","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.347258258Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"client":"2","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.347285101Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.347312936Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.347346101Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"ds_connection connected from Ok(127.0.0.1:44930)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.347530747Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"ds_connection connected from Ok(127.0.0.1:48731)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.347566758Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"ds_connection connected from Ok(127.0.0.1:42666)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.347603551Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.35033562Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"client":"1","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.35082549Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"client":"0","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.350911721Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"client":"2","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.351060926Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.351563431Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"[0]R flush_numbers[0..12]: [111, 112, 125, 127, 127, 126, 126, 125, 125, 124, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.351750472Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"downstairs","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"[0]R generation[0..12]: [60, 61, 66, 67, 67, 66, 66, 66, 66, 66, 66, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.351789549Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"downstairs","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.35182533Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"downstairs","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"[1]R flush_numbers[0..12]: [111, 112, 125, 127, 127, 126, 126, 125, 125, 124, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.351859327Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"downstairs","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"[1]R generation[0..12]: [60, 61, 66, 67, 67, 66, 66, 66, 66, 66, 66, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.351892062Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"downstairs","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.351927312Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"downstairs","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"[2]R flush_numbers[0..12]: [111, 112, 125, 127, 127, 126, 126, 125, 125, 124, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.351961369Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"downstairs","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"[2]R generation[0..12]: [60, 61, 66, 67, 67, 66, 66, 66, 66, 66, 66, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.351994054Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"downstairs","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352038311Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"downstairs","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.35206773Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"downstairs","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352097278Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"downstairs","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352125264Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"downstairs","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352153479Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"":"downstairs","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352183829Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352210552Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"client":"0","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352239971Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"client":"1","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352269289Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"client":"2","":"downstairs","session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352299619Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"session_id":"Wait for a query_work_queue command to finish before sending IO dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352343395Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"fd0eda8d-bc68-4d66-9a16-ca2fef446f0c is now active with session: dd04104d-b4ab-4dfc-bd5b-2905aa60dd36","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352375409Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352402212Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464,"session_id":"dd04104d-b4ab-4dfc-bd5b-2905aa60dd36"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352430167Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464} {"msg":"Activated sub_volume fd0eda8d-bc68-4d66-9a16-ca2fef446f0c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:04.352454435Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 445 len 28672 data: 7 7 5 6 6 5 4 02/30 Read block 382 len 32768 03/30 Read block 564 len 32768 04/30 Write block 6 len 36864 data: 11 11 7 8 5 5 4 5 4 05/30 Read block 593 len 12288 06/30 Read block 205 len 16384 07/30 Write block 548 len 28672 data: 6 5 7 9 10 9 11 08/30 Read block 529 len 4096 09/30 Write block 104 len 24576 data: 4 5 4 4 4 4 10/30 Write block 185 len 8192 data: 6 7 11/30 Read block 158 len 8192 12/30 Read block 258 len 32768 13/30 Read block 587 len 20480 14/30 Flush 15/30 Write block 2 len 12288 data: 4 6 8 16/30 Write block 408 len 20480 data: 7 6 5 4 3 17/30 Flush 18/30 Write block 129 len 16384 data: 7 7 6 6 19/30 Read block 207 len 8192 20/30 Read block 445 len 32768 21/30 Read block 576 len 40960 22/30 Read block 464 len 32768 23/30 Write block 3 len 4096 data: 7 24/30 Read block 111 len 8192 25/30 Write block 353 len 24576 data: 3 2 4 6 8 10 26/30 Read block 325 len 32768 27/30 Write block 183 len 12288 data: 7 6 7 28/30 Read block 105 len 8192 29/30 Read block 151 len 12288 30/30 Read block 31 len 12288 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 8 Done Done Sent false 1002 Acked Read 8 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Read 4 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 8 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Read 8 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Write 6 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:04.390851727Z","hostname":"bmat-EVT22200007-0000ead9","pid":1464} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 68 68 60 129 129 111 T T F <--- 1 020-039 61 61 61 112 112 112 F F F 2 040-059 66 66 66 125 125 125 F F F 3 060-079 67 67 67 127 127 127 F F F 4 080-099 67 67 67 127 127 127 F F F 5 100-119 68 68 66 128 128 126 F F F <--- 6 120-139 66 66 66 126 126 126 T T F <--- 7 140-159 66 66 66 125 125 125 F F F 8 160-179 66 66 66 125 125 125 F F F 9 180-199 68 68 66 128 128 124 T T F <--- 10 200-219 66 66 66 124 124 124 F F F 11 220-239 64 64 64 120 120 120 F F F 12 240-259 64 64 64 117 117 117 F F F 13 260-279 67 67 67 127 127 127 F F F 14 280-299 65 65 65 122 122 122 F F F 15 300-319 66 66 66 125 125 125 F F F 16 320-339 66 66 66 125 125 125 F F F 17 340-359 63 63 63 116 116 116 T T F <--- 18 360-379 66 66 66 125 125 125 F F F 19 380-399 64 64 64 119 119 119 F F F 20 400-419 68 68 58 129 129 104 F F F <--- 21 420-439 60 60 60 107 107 107 F F F 22 440-459 68 68 64 128 128 121 F F T <--- 23 460-479 66 66 66 123 123 123 F F F 24 480-499 64 64 64 120 120 120 F F F 25 500-519 64 64 64 119 119 119 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 68 68 66 128 128 124 F F F <--- 28 560-579 58 58 58 105 105 105 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 68, Max flush: 129 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 130 130 130 F F F 1 020-039 61 61 61 112 112 112 F F F 2 040-059 66 66 66 125 125 125 F F F 3 060-079 67 67 67 127 127 127 F F F 4 080-099 67 67 67 127 127 127 F F F 5 100-119 68 68 68 128 128 128 F F F 6 120-139 69 69 69 130 130 130 F F F 7 140-159 66 66 66 125 125 125 F F F 8 160-179 66 66 66 125 125 125 F F F 9 180-199 69 69 69 130 130 130 F F F 10 200-219 66 66 66 124 124 124 F F F 11 220-239 64 64 64 120 120 120 F F F 12 240-259 64 64 64 117 117 117 F F F 13 260-279 67 67 67 127 127 127 F F F 14 280-299 65 65 65 122 122 122 F F F 15 300-319 66 66 66 125 125 125 F F F 16 320-339 66 66 66 125 125 125 F F F 17 340-359 69 69 69 130 130 130 F F F 18 360-379 66 66 66 125 125 125 F F F 19 380-399 64 64 64 119 119 119 F F F 20 400-419 68 68 68 129 129 129 F F F 21 420-439 60 60 60 107 107 107 F F F 22 440-459 68 68 68 128 128 128 F F F 23 460-479 66 66 66 123 123 123 F F F 24 480-499 64 64 64 120 120 120 F F F 25 500-519 64 64 64 119 119 119 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 68 68 68 128 128 128 F F F 28 560-579 58 58 58 105 105 105 F F F 29 580-599 53 53 53 93 93 93 F F F Max gen: 69, Max flush: 130 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:08.990952748Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.020157259Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.020913737Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.02105186Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.021087982Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.021133983Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472} {"msg":"Crucible a3183ca5-0d65-49fd-aa51-3c3f94eea4ba has session id: d4e78559-66b2-4141-b869-eac60f5d38cd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.022480789Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"Upstairs opts: Upstairs UUID: a3183ca5-0d65-49fd-aa51-3c3f94eea4ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.022552421Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"Crucible stats registered with UUID: a3183ca5-0d65-49fd-aa51-3c3f94eea4ba","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.022585847Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:09.022613562Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.023079254Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.023139053Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.023169022Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.023520247Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.023552361Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472} {"msg":"a3183ca5-0d65-49fd-aa51-3c3f94eea4ba active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.023773038Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.023802987Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"client":"0","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.023838497Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"client":"1","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.023866753Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"client":"2","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.023894679Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.024109564Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.024144674Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:35211)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.024399388Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:56204)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.02461761Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:60571)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.02466832Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.026826974Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"client":"2","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.027149212Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"client":"0","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.027399939Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"client":"1","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.027659913Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.027882404Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"[0]R flush_numbers[0..12]: [130, 112, 125, 127, 127, 128, 130, 125, 125, 130, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.028615926Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"downstairs","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"[0]R generation[0..12]: [69, 61, 66, 67, 67, 68, 69, 66, 66, 69, 66, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.028711966Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"downstairs","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.02876917Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"downstairs","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"[1]R flush_numbers[0..12]: [130, 112, 125, 127, 127, 128, 130, 125, 125, 130, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.028813688Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"downstairs","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"[1]R generation[0..12]: [69, 61, 66, 67, 67, 68, 69, 66, 66, 69, 66, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.028859599Z","hostname":"Wait for a query_work_queue command to finish before sending IO bmat-EVT22200007-0000ead9","pid":1472,"":"downstairs","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.028925629Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"downstairs","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"[2]R flush_numbers[0..12]: [130, 112, 125, 127, 127, 128, 130, 125, 125, 130, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.028975809Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"downstairs","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"[2]R generation[0..12]: [69, 61, 66, 67, 67, 68, 69, 66, 66, 69, 66, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029021229Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"downstairs","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029071809Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"downstairs","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.02911271Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"downstairs","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029152288Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"downstairs","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029189762Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"downstairs","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029237888Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"":"downstairs","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029285632Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 name":"crucible","level":30,"time":"2026-03-04T01:11:09.029328888Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"client":"0","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029374177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"client":"1","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029423425Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"client":"2","":"downstairs","session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029468785Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029510117Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"a3183ca5-0d65-49fd-aa51-3c3f94eea4ba is now active with session: d4e78559-66b2-4141-b869-eac60f5d38cd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029553132Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029588211Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472,"session_id":"d4e78559-66b2-4141-b869-eac60f5d38cd"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029622369Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472} {"msg":"Activated sub_volume a3183ca5-0d65-49fd-aa51-3c3f94eea4ba","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:09.029654853Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 362 len 8192 data: 10 8 02/30 Write block 236 len 32768 data: 8 8 8 10 9 9 7 6 03/30 Write block 303 len 36864 data: 5 7 6 7 9 9 9 9 9 04/30 Write block 400 len 20480 data: 8 9 9 8 9 05/30 Write block 189 len 24576 data: 8 7 9 8 10 9 06/30 Write block 305 len 20480 data: 7 8 10 10 10 07/30 Write block 495 len 12288 data: 6 6 6 08/30 Write block 205 len 20480 data: 7 7 8 7 7 09/30 Read block 258 len 16384 10/30 Read block 173 len 12288 11/30 Read block 335 len 16384 12/30 Write block 314 len 40960 data: 4 6 8 7 6 6 10 9 10 9 13/30 Read block 115 len 40960 14/30 Read block 96 len 16384 15/30 Flush 16/30 Write block 584 len 36864 data: 7 6 5 3 3 3 4 3 4 17/30 Flush 18/30 Write block 449 len 12288 data: 7 6 5 19/30 Write block 510 len 32768 data: 8 7 7 7 8 7 6 6 20/30 Read block 129 len 4096 21/30 Write block 373 len 8192 data: 5 5 22/30 Read block 241 len 8192 23/30 Write block 60 len 20480 data: 4 4 4 4 6 24/30 Read block 297 len 12288 25/30 Read block 417 len 40960 26/30 Read block 120 len 24576 27/30 Write block 404 len 20480 data: 10 10 9 8 8 28/30 Read block 65 len 32768 29/30 Write block 565 len 32768 data: 6 6 5 6 6 8 7 8 30/30 Write block 391 len 28672 data: 6 7 5 6 5 5 6 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Sent Done false 1001 Acked Write 8 Done Sent Done false 1002 Acked Write 9 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) None Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:09.062552296Z","hostname":"bmat-EVT22200007-0000ead9","pid":1472} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 130 130 130 F F F 1 020-039 61 61 61 112 112 112 F F F 2 040-059 66 66 66 125 125 125 F F F 3 060-079 67 67 67 127 127 127 T F T <--- 4 080-099 67 67 67 127 127 127 F F F 5 100-119 68 68 68 128 128 128 F F F 6 120-139 69 69 69 130 130 130 F F F 7 140-159 66 66 66 125 125 125 F F F 8 160-179 66 66 66 125 125 125 F F F 9 180-199 70 69 70 131 130 131 F F F <--- 10 200-219 70 66 70 131 124 131 F F F <--- 11 220-239 70 64 70 131 120 131 F F F <--- 12 240-259 70 64 70 131 117 131 F F F <--- 13 260-279 67 67 67 127 127 127 F F F 14 280-299 65 65 65 122 122 122 F F F 15 300-319 70 66 70 131 125 131 F F F <--- 16 320-339 70 66 70 131 125 131 F F F <--- 17 340-359 69 69 69 130 130 130 F F F 18 360-379 70 66 70 131 125 131 T F T <--- 19 380-399 64 64 64 119 119 119 T F T <--- 20 400-419 70 68 70 131 129 131 T F T <--- 21 420-439 60 60 60 107 107 107 F F F 22 440-459 68 68 68 128 128 128 T F T <--- 23 460-479 66 66 66 123 123 123 F F F 24 480-499 70 64 70 131 120 131 F F F <--- 25 500-519 64 64 64 119 119 119 T F T <--- 26 520-539 62 62 62 114 114 114 F F F 27 540-559 68 68 68 128 128 128 F F F 28 560-579 58 58 58 105 105 105 T F T <--- 29 580-599 70 53 70 132 93 132 F F F <--- Max gen: 70, Max flush: 132 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 130 130 130 F F F 1 020-039 61 61 61 112 112 112 F F F 2 040-059 66 66 66 125 125 125 F F F 3 060-079 71 71 71 133 133 133 F F F 4 080-099 67 67 67 127 127 127 F F F 5 100-119 68 68 68 128 128 128 F F F 6 120-139 69 69 69 130 130 130 F F F 7 140-159 66 66 66 125 125 125 F F F 8 160-179 66 66 66 125 125 125 F F F 9 180-199 70 70 70 131 131 131 F F F 10 200-219 70 70 70 131 131 131 F F F 11 220-239 70 70 70 131 131 131 F F F 12 240-259 70 70 70 131 131 131 F F F 13 260-279 67 67 67 127 127 127 F F F 14 280-299 65 65 65 122 122 122 F F F 15 300-319 70 70 70 131 131 131 F F F 16 320-339 70 70 70 131 131 131 F F F 17 340-359 69 69 69 130 130 130 F F F 18 360-379 71 71 71 133 133 133 F F F 19 380-399 71 71 71 133 133 133 F F F 20 400-419 71 71 71 133 133 133 F F F 21 420-439 60 60 60 107 107 107 F F F 22 440-459 71 71 71 133 133 133 F F F 23 460-479 66 66 66 123 123 123 F F F 24 480-499 70 70 70 131 131 131 F F F 25 500-519 71 71 71 133 133 133 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 68 68 68 128 128 128 F F F 28 560-579 71 71 71 133 133 133 F F F 29 580-599 70 70 70 132 132 132 F F F Max gen: 71, Max flush: 133 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.735113689Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.762822169Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.763490091Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.763575159Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.763599738Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.763635389Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480} {"msg":"Crucible 23fd9268-f16b-491f-9a6c-dc17f0f06480 has session id: 4a659fe4-5ce6-4752-b2f0-d02ed5129566","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.764907177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Upstairs opts: Upstairs UUID: 23fd9268-f16b-491f-9a6c-dc17f0f06480, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.765027575Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Crucible stats registered with UUID: 23fd9268-f16b-491f-9a6c-dc17f0f06480","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.76509083Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:13.765129817Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.765674365Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.765754073Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.765791758Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.76615626Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.766198664Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480} {"msg":"23fd9268-f16b-491f-9a6c-dc17f0f06480 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.766353901Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.766397056Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"client":"0","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.766435883Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"client":"1","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.766473197Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"client":"2","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.766509349Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.766547484Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.766782479Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"ds_connection connected from Ok(127.0.0.1:53392)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.76723755Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"ds_connection connected from Ok(127.0.0.1:63875)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.767304353Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"ds_connection connected from Ok(127.0.0.1:50179)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.76737393Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.768770566Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"client":"1","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.769453227Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"client":"2","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.769529969Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"client":"0","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.769846135Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.770229013Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"[0]R flush_numbers[0..12]: [130, 112, 125, 133, 127, 128, 130, 125, 125, 131, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.770521923Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"downstairs","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"[0]R generation[0..12]: [69, 61, 66, 71, 67, 68, 69, 66, 66, 70, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.770574096Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"downstairs","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.77063668Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"downstairs","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"[1]R flush_numbers[0..12]: [130, 112, 125, 133, 127, 128, 130, 125, 125, 131, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.770698763Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"downstairs","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"[1]R generation[0..12]: [69, 61, 66, 71, 67, 68, 69, 66, 66, 70, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.770758241Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"downstairs","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.770822047Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"downstairs","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"[2]R flush_numbers[0..12]: [130, 112, 125, 133, 127, 128, 130, 125, 125, 131, 131, 131]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.770883018Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"downstairs","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R generation[0..12]: [69, 61, 66, 71, 67, 68, 69, 66, 66, 70, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.770952325Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"downstairs","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.77101015Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"downstairs","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.771062433Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"downstairs","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.771117322Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"downstairs","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Next flush: 134","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.771168904Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"downstairs","":"downstairs","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.771229023Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"":"downstairs","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.771284733Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.77133382Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"client":"0","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.771387587Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"client":"1","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.771538405Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"client":"2","":"downstairs","session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.771576731Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.771629486Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"23fd9268-f16b-491f-9a6c-dc17f0f06480 is now active with session: 4a659fe4-5ce6-4752-b2f0-d02ed5129566","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.771687811Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.771736337Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480,"session_id":"4a659fe4-5ce6-4752-b2f0-d02ed5129566"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.771787539Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480} {"msg":"Activated sub_volume 23fd9268-f16b-491f-9a6c-dc17f0f06480","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:13.771831315Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 206 len 32768 data: 8 9 8 8 8 7 6 3 02/30 Write block 411 len 24576 data: 5 4 3 2 3 3 03/30 Write block 495 len 4096 data: 7 04/30 Read block 590 len 4096 05/30 Write block 300 len 28672 data: 4 4 4 6 8 8 9 06/30 Write block 93 len 24576 data: 8 7 6 6 7 6 07/30 Write block 158 len 4096 data: 5 08/30 Write block 435 len 24576 data: 8 10 12 13 9 8 09/30 Write block 107 len 8192 data: 5 5 10/30 Read block 78 len 36864 11/30 Write block 137 len 16384 data: 8 9 10 9 12/30 Write block 168 len 24576 data: 11 7 6 6 4 4 13/30 Read block 327 len 40960 14/30 Write block 33 len 4096 data: 5 15/30 Read block 36 len 8192 16/30 Flush 17/30 Write block 34 len 4096 data: 5 18/30 Read block 5 len 24576 19/30 Read block 418 len 24576 20/30 Write block 155 len 12288 data: 4 5 6 21/30 Write block 461 len 40960 data: 6 9 8 7 6 7 6 6 6 6 22/30 Read block 77 len 28672 23/30 Write block 330 len 40960 data: 4 3 3 3 3 4 7 7 5 5 24/30 Read block 9 len 12288 25/30 Read block 588 len 40960 26/30 Write block 566 len 16384 data: 7 6 7 7 27/30 Write block 586 len 4096 data: 6 28/30 Write block 180 len 20480 data: 8 7 7 8 7 29/30 Read block 496 len 32768 30/30 Write block 215 len 24576 data: 3 5 7 7 4 4 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Sent false 1001 Acked Write 6 Done Done Sent false 1002 Acked Write 1 Done Done Sent false 1003 Acked Read 1 Done Done Sent false 1004 Acked Write 7 Done Done Sent false 1005 Acked Write 6 Done Done Sent false 1006 Acked Write 1 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:13.801209069Z","hostname":"bmat-EVT22200007-0000ead9","pid":1480} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 130 130 130 F F F 1 020-039 72 72 61 134 134 112 T T F <--- 2 040-059 66 66 66 125 125 125 F F F 3 060-079 71 71 71 133 133 133 F F F 4 080-099 72 72 67 134 134 127 F F F <--- 5 100-119 72 72 68 134 134 128 F F F <--- 6 120-139 72 72 69 134 134 130 F F F <--- 7 140-159 72 72 66 134 134 125 T T F <--- 8 160-179 72 72 66 134 134 125 F F F <--- 9 180-199 70 70 70 131 131 131 T T F <--- 10 200-219 72 72 70 134 134 131 T T F <--- 11 220-239 70 70 70 131 131 131 T T F <--- 12 240-259 70 70 70 131 131 131 F F F 13 260-279 67 67 67 127 127 127 F F F 14 280-299 65 65 65 122 122 122 F F F 15 300-319 72 72 70 134 134 131 F F F <--- 16 320-339 70 70 70 131 131 131 T T F <--- 17 340-359 69 69 69 130 130 130 F F F 18 360-379 71 71 71 133 133 133 F F F 19 380-399 71 71 71 133 133 133 F F F 20 400-419 72 72 71 134 134 133 F F F <--- 21 420-439 72 72 60 134 134 107 F F F <--- 22 440-459 72 72 71 134 134 133 F F F <--- 23 460-479 66 66 66 123 123 123 T T F <--- 24 480-499 72 72 70 134 134 131 F F F <--- 25 500-519 71 71 71 133 133 133 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 68 68 68 128 128 128 F F F 28 560-579 71 71 71 133 133 133 T T F <--- 29 580-599 70 70 70 132 132 132 T T F <--- Max gen: 72, Max flush: 134 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 130 130 130 F F F 1 020-039 73 73 73 135 135 135 F F F 2 040-059 66 66 66 125 125 125 F F F 3 060-079 71 71 71 133 133 133 F F F 4 080-099 72 72 72 134 134 134 F F F 5 100-119 72 72 72 134 134 134 F F F 6 120-139 72 72 72 134 134 134 F F F 7 140-159 73 73 73 135 135 135 F F F 8 160-179 72 72 72 134 134 134 F F F 9 180-199 73 73 73 135 135 135 F F F 10 200-219 73 73 73 135 135 135 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 70 70 70 131 131 131 F F F 13 260-279 67 67 67 127 127 127 F F F 14 280-299 65 65 65 122 122 122 F F F 15 300-319 72 72 72 134 134 134 F F F 16 320-339 73 73 73 135 135 135 F F F 17 340-359 69 69 69 130 130 130 F F F 18 360-379 71 71 71 133 133 133 F F F 19 380-399 71 71 71 133 133 133 F F F 20 400-419 72 72 72 134 134 134 F F F 21 420-439 72 72 72 134 134 134 F F F 22 440-459 72 72 72 134 134 134 F F F 23 460-479 73 73 73 135 135 135 F F F 24 480-499 72 72 72 134 134 134 F F F 25 500-519 71 71 71 133 133 133 F F F 26 520-539 62 62 62 114 114 114 F F F 27 540-559 68 68 68 128 128 128 F F F 28 560-579 73 73 73 135 135 135 F F F 29 580-599 73 73 73 135 135 135 F F F Max gen: 73, Max flush: 135 Begin loop 37 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.500884936Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.529680627Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.530477405Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.530658894Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.530716188Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.530775745Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488} {"msg":"Crucible 53743674-8520-435a-98a3-a0bb74e43d6e has session id: 50cc33de-a51c-4b6c-aa52-3cf737d19a9f","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.531481733Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"Upstairs opts: Upstairs UUID: 53743674-8520-435a-98a3-a0bb74e43d6e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.531619376Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"Crucible stats registered with UUID: 53743674-8520-435a-98a3-a0bb74e43d6e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.531668693Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:18.53170754Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.532008877Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.532120468Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.532164224Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.532588454Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.532641459Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488} {"msg":"53743674-8520-435a-98a3-a0bb74e43d6e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.53269225Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.532735325Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"client":"0","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.532777018Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"client":"1","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.532817107Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"client":"2","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.532856625Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.532893939Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62946)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.533099957Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40987)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.533140828Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.533169334Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57340)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.533527724Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.535029948Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"client":"2","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.535701438Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"client":"0","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.535840894Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.535884079Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"client":"1","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.53641996Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"[0]R flush_numbers[0..12]: [130, 135, 125, 133, 134, 134, 134, 135, 134, 135, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.536882696Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"downstairs","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"[0]R generation[0..12]: [69, 73, 66, 71, 72, 72, 72, 73, 72, 73, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.536956162Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"downstairs","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537021321Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"downstairs","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"[1]R flush_numbers[0..12]: [130, 135, 125, 133, 134, 134, 134, 135, 134, 135, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537057623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"downstairs","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"[1]R generation[0..12]: [69, 73, 66, 71, 72, 72, 72, 73, 72, 73, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537090087Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"downstairs","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537125317Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"downstairs","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"[2]R flush_numbers[0..12]: [130, 135, 125, 133, 134, 134, 134, 135, 134, 135, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537159084Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"downstairs","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"[2]R generation[0..12]: [69, 73, 66, 71, 72, 72, 72, 73, 72, 73, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537191738Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"downstairs","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537227359Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"downstairs","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537256186Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"downstairs","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:11:18.537286215Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"downstairs","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537323469Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"downstairs","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537351675Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"":"downstairs","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537381654Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537408197Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"client":"0","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537442314Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"client":"1","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537472043Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"client":"2","":"downstairs","session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537502684Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537530749Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"53743674-8520-435a-98a3-a0bb74e43d6e is now active with session: 50cc33de-a51c-4b6c-aa52-3cf737d19a9f","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537562502Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537597802Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488,"session_id":"50cc33de-a51c-4b6c-aa52-3cf737d19a9f"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537625968Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488} {"msg":"Activated sub_volume 53743674-8520-435a-98a3-a0bb74e43d6e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:18.537650647Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 401 len 4096 data: 10 02/30 Read block 29 len 16384 03/30 Write block 194 len 20480 data: 10 8 9 8 8 04/30 Read block 309 len 20480 05/30 Write block 471 len 28672 data: 7 7 7 8 8 7 7 06/30 Read block 589 len 16384 07/30 Write block 365 len 16384 data: 7 5 5 5 08/30 Read block 524 len 12288 09/30 Write block 298 len 20480 data: 6 4 5 5 5 10/30 Read block 451 len 20480 11/30 Write block 42 len 8192 data: 3 4 12/30 Read block 178 len 4096 13/30 Read block 508 len 40960 14/30 Write block 165 len 40960 data: 11 8 9 12 8 7 7 5 5 5 15/30 Flush 16/30 Write block 529 len 16384 data: 7 6 6 6 17/30 Read block 122 len 4096 18/30 Write block 356 len 32768 data: 7 9 11 10 9 9 11 9 19/30 Read block 401 len 12288 20/30 Write block 189 len 40960 data: 9 8 10 9 11 11 9 10 9 9 21/30 Write block 91 len 28672 data: 6 6 9 8 7 7 8 22/30 Read block 283 len 12288 23/30 Read block 125 len 4096 24/30 Read block 481 len 28672 25/30 Write block 378 len 36864 data: 5 6 5 7 7 5 5 7 5 26/30 Write block 113 len 36864 data: 11 10 9 10 10 9 8 9 6 27/30 Read block 571 len 4096 28/30 Write block 257 len 28672 data: 6 5 6 9 10 9 10 29/30 Write block 248 len 32768 data: 7 10 9 5 6 5 6 7 30/30 Read block 548 len 12288 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Write 4 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 0 29 Done 2 29 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:18.575990965Z","hostname":"bmat-EVT22200007-0000ead9","pid":1488} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 130 130 130 F F F 1 020-039 73 73 73 135 135 135 F F F 2 040-059 66 74 74 125 136 136 F F F <--- 3 060-079 71 71 71 133 133 133 F F F 4 080-099 72 72 72 134 134 134 F T T <--- 5 100-119 72 72 72 134 134 134 F T T <--- 6 120-139 72 72 72 134 134 134 F T T <--- 7 140-159 73 73 73 135 135 135 F F F 8 160-179 72 74 74 134 136 136 F F F <--- 9 180-199 73 74 74 135 136 136 F T T <--- 10 200-219 73 73 73 135 135 135 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 70 70 70 131 131 131 F T T <--- 13 260-279 67 67 67 127 127 127 F T T <--- 14 280-299 65 74 74 122 136 136 F F F <--- 15 300-319 72 74 74 134 136 136 F F F <--- 16 320-339 73 73 73 135 135 135 F F F 17 340-359 69 69 69 130 130 130 F T T <--- 18 360-379 71 74 74 133 136 136 F T T <--- 19 380-399 71 71 71 133 133 133 F T T <--- 20 400-419 72 74 74 134 136 136 T F F <--- 21 420-439 72 72 72 134 134 134 F F F 22 440-459 72 72 72 134 134 134 F F F 23 460-479 73 74 74 135 136 136 F F F <--- 24 480-499 72 72 72 134 134 134 F F F 25 500-519 71 71 71 133 133 133 F F F 26 520-539 62 62 62 114 114 114 F T T <--- 27 540-559 68 68 68 128 128 128 F F F 28 560-579 73 73 73 135 135 135 F F F 29 580-599 73 73 73 135 135 135 F F F Max gen: 74, Max flush: 136 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 130 130 130 F F F 1 020-039 73 73 73 135 135 135 F F F 2 040-059 74 74 74 136 136 136 F F F 3 060-079 71 71 71 133 133 133 F F F 4 080-099 75 75 75 137 137 137 F F F 5 100-119 75 75 75 137 137 137 F F F 6 120-139 75 75 75 137 137 137 F F F 7 140-159 73 73 73 135 135 135 F F F 8 160-179 74 74 74 136 136 136 F F F 9 180-199 75 75 75 137 137 137 F F F 10 200-219 73 73 73 135 135 135 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 75 75 75 137 137 137 F F F 13 260-279 75 75 75 137 137 137 F F F 14 280-299 74 74 74 136 136 136 F F F 15 300-319 74 74 74 136 136 136 F F F 16 320-339 73 73 73 135 135 135 F F F 17 340-359 75 75 75 137 137 137 F F F 18 360-379 75 75 75 137 137 137 F F F 19 380-399 75 75 75 137 137 137 F F F 20 400-419 74 74 74 136 136 136 F F F 21 420-439 72 72 72 134 134 134 F F F 22 440-459 72 72 72 134 134 134 F F F 23 460-479 74 74 74 136 136 136 F F F 24 480-499 72 72 72 134 134 134 F F F 25 500-519 71 71 71 133 133 133 F F F 26 520-539 75 75 75 137 137 137 F F F 27 540-559 68 68 68 128 128 128 F F F 28 560-579 73 73 73 135 135 135 F F F 29 580-599 73 73 73 135 135 135 F F F Max gen: 75, Max flush: 137 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.260473972Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:23.287395302Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.287483617Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.313046739Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.313731554Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.313832264Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.31386576Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.31391125Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496} {"msg":"Crucible 7171d484-0f1d-4798-bf8a-62342ae63aaa has session id: aee3c85a-74a5-433e-a71e-c0621baaaef3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.315092218Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"Upstairs opts: Upstairs UUID: 7171d484-0f1d-4798-bf8a-62342ae63aaa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.315220192Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"Crucible stats registered with UUID: 7171d484-0f1d-4798-bf8a-62342ae63aaa","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.315257175Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:23.315286032Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.315850119Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.315950578Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.315984084Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.316177978Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.316215142Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496} {"msg":"7171d484-0f1d-4798-bf8a-62342ae63aaa active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.316398675Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.316443945Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"client":"0","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.316477461Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"client":"1","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.316509955Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"client":"2","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.316539173Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.316741253Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.316792394Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"ds_connection connected from Ok(127.0.0.1:34058)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.316833376Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"ds_connection connected from Ok(127.0.0.1:34997)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.316871251Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"ds_connection connected from Ok(127.0.0.1:38310)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.317052069Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.318541487Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"client":"1","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.318629121Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"client":"2","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.319403224Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.319454074Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.319949766Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"client":"0","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"[0]R flush_numbers[0..12]: [130, 135, 136, 133, 137, 137, 137, 135, 136, 137, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321220852Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"downstairs","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"[0]R generation[0..12]: [69, 73, 74, 71, 75, 75, 75, 73, 74, 75, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321264899Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"downstairs","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.32130071Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"downstairs","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"[1]R flush_numbers[0..12]: [130, 135, 136, 133, 137, 137, 137, 135, 136, 137, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321334617Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"downstairs","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"[1]R generation[0..12]: [69, 73, 74, 71, 75, 75, 75, 73, 74, 75, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321367222Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"downstairs","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321402061Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"downstairs","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"[2]R flush_numbers[0..12]: [130, 135, 136, 133, 137, 137, 137, 135, 136, 137, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321443273Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"downstairs","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"[2]R generation[0..12]: [69, 73, 74, 71, 75, 75, 75, 73, 74, 75, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321476248Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"downstairs","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321511498Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"downstairs","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321540004Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"downstairs","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321569943Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"downstairs","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321598139Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"downstairs","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321626345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"":"downstairs","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321655904Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321682887Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"client":"0","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321712876Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"client":"1","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321742164Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"client":"2","":"downstairs","session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321772534Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.32180062Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"7171d484-0f1d-4798-bf8a-62342ae63aaa is now active with session: aee3c85a-74a5-433e-a71e-c0621baaaef3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321832664Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321859416Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496,"session_id":"aee3c85a-74a5-433e-a71e-c0621baaaef3"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.32189172Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496} {"msg":"Activated sub_volume 7171d484-0f1d-4798-bf8a-62342ae63aaa","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:23.321925517Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 193 len 8192 data: 12 12 02/30 Read block 189 len 36864 03/30 Flush 04/30 Flush 05/30 Flush 06/30 Flush 07/30 Write block 406 len 16384 data: 10 9 9 7 08/30 Read block 577 len 16384 09/30 Read block 281 len 28672 10/30 Read block 132 len 40960 11/30 Write block 366 len 40960 data: 6 6 6 5 5 5 5 6 6 3 12/30 Write block 108 len 8192 data: 6 5 13/30 Read block 23 len 16384 14/30 Read block 352 len 8192 15/30 Flush 16/30 Write block 172 len 36864 data: 6 6 6 5 7 7 4 5 9 17/30 Read block 580 len 28672 18/30 Read block 463 len 40960 19/30 Read block 558 len 12288 20/30 Flush 21/30 Write block 504 len 28672 data: 7 6 5 5 6 6 9 22/30 Flush 23/30 Write block 323 len 20480 data: 10 11 6 5 5 24/30 Write block 582 len 12288 data: 6 6 8 25/30 Read block 469 len 32768 26/30 Read block 97 len 40960 27/30 Read block 84 len 20480 28/30 Write block 556 len 40960 data: 10 8 7 7 6 5 6 7 8 7 29/30 Write block 117 len 24576 data: 11 10 9 10 7 5 30/30 Write block 151 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 4 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 9 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 24 3 3 30 Done 0 21 21 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:23.358375463Z","hostname":"bmat-EVT22200007-0000ead9","pid":1496} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 130 130 130 F F F 1 020-039 73 73 73 135 135 135 F F F 2 040-059 74 74 74 136 136 136 F F F 3 060-079 71 71 71 133 133 133 F F F 4 080-099 75 75 75 137 137 137 F F F 5 100-119 75 76 76 137 142 142 F T T <--- 6 120-139 75 75 75 137 137 137 F T T <--- 7 140-159 73 73 73 135 135 135 F T T <--- 8 160-179 74 76 76 136 143 143 F F F <--- 9 180-199 76 76 76 138 143 143 F F F <--- 10 200-219 73 73 73 135 135 135 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 75 75 75 137 137 137 F F F 13 260-279 75 75 75 137 137 137 F F F 14 280-299 74 74 74 136 136 136 F F F 15 300-319 74 74 74 136 136 136 F F F 16 320-339 73 73 73 135 135 135 F T T <--- 17 340-359 75 75 75 137 137 137 F F F 18 360-379 75 76 76 137 142 142 F F F <--- 19 380-399 75 75 75 137 137 137 F F F 20 400-419 74 76 76 136 142 142 F F F <--- 21 420-439 72 72 72 134 134 134 F F F 22 440-459 72 72 72 134 134 134 F F F 23 460-479 74 74 74 136 136 136 F F F 24 480-499 72 72 72 134 134 134 F F F 25 500-519 71 76 76 133 144 144 F F F <--- 26 520-539 75 75 75 137 137 137 F F F 27 540-559 68 68 68 128 128 128 F T T <--- 28 560-579 73 73 73 135 135 135 F T T <--- 29 580-599 73 73 73 135 135 135 F T T <--- Max gen: 76, Max flush: 144 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 130 130 130 F F F 1 020-039 73 73 73 135 135 135 F F F 2 040-059 74 74 74 136 136 136 F F F 3 060-079 71 71 71 133 133 133 F F F 4 080-099 75 75 75 137 137 137 F F F 5 100-119 77 77 77 145 145 145 F F F 6 120-139 77 77 77 145 145 145 F F F 7 140-159 77 77 77 145 145 145 F F F 8 160-179 76 76 76 143 143 143 F F F 9 180-199 76 76 76 143 143 143 F F F 10 200-219 73 73 73 135 135 135 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 75 75 75 137 137 137 F F F 13 260-279 75 75 75 137 137 137 F F F 14 280-299 74 74 74 136 136 136 F F F 15 300-319 74 74 74 136 136 136 F F F 16 320-339 77 77 77 145 145 145 F F F 17 340-359 75 75 75 137 137 137 F F F 18 360-379 76 76 76 142 142 142 F F F 19 380-399 75 75 75 137 137 137 F F F 20 400-419 76 76 76 142 142 142 F F F 21 420-439 72 72 72 134 134 134 F F F 22 440-459 72 72 72 134 134 134 F F F 23 460-479 74 74 74 136 136 136 F F F 24 480-499 72 72 72 134 134 134 F F F 25 500-519 76 76 76 144 144 144 F F F 26 520-539 75 75 75 137 137 137 F F F 27 540-559 77 77 77 145 145 145 F F F 28 560-579 77 77 77 145 145 145 F F F 29 580-599 77 77 77 145 145 145 F F F Max gen: 77, Max flush: 145 Begin loop 39 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.020561114Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:28.047187571Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.047273601Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.080945748Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.08132591Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.081392913Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.081418474Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.081454214Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504} {"msg":"Crucible eebb3a95-8ca3-4409-baa5-9e8f2be5e78e has session id: b10af835-b8ae-4dc0-94e4-e6446dd83177","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.082612808Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"Upstairs opts: Upstairs UUID: eebb3a95-8ca3-4409-baa5-9e8f2be5e78e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.082672005Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"Crucible stats registered with UUID: eebb3a95-8ca3-4409-baa5-9e8f2be5e78e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.082700472Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:28.082725882Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.083210291Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.083266492Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.083301842Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.083658298Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.083693197Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504} {"msg":"eebb3a95-8ca3-4409-baa5-9e8f2be5e78e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.083854266Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.083883945Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"client":"0","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.083911369Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"client":"1","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.083938262Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"client":"2","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.084060815Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.084094452Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.084122136Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"ds_connection connected from Ok(127.0.0.1:65216)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.0847301Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"ds_connection connected from Ok(127.0.0.1:37450)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.084808485Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"ds_connection connected from Ok(127.0.0.1:39901)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.084839867Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.086673357Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"client":"2","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.086754097Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"client":"1","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.086789848Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"client":"0","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087420717Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087465506Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"[0]R flush_numbers[0..12]: [130, 135, 136, 133, 137, 145, 145, 145, 143, 143, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087591485Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"downstairs","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"[0]R generation[0..12]: [69, 73, 74, 71, 75, 77, 77, 77, 76, 76, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.08762948Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"downstairs","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087661895Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"downstairs","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"[1]R flush_numbers[0..12]: [130, 135, 136, 133, 137, 145, 145, 145, 143, 143, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087692896Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"downstairs","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"[1]R generation[0..12]: [69, 73, 74, 71, 75, 77, 77, 77, 76, 76, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087722986Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"downstairs","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087754438Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"downstairs","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"[2]R flush_numbers[0..12]: [130, 135, 136, 133, 137, 145, 145, 145, 143, 143, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087785299Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"downstairs","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"[2]R generation[0..12]: [69, 73, 74, 71, 75, 77, 77, 77, 76, 76, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087814857Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"downstairs","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.08784641Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"downstairs","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087872131Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"downstairs","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087899174Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"downstairs","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087924895Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"downstairs","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087950646Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"":"downstairs","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.087983491Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-04T01:11:28.088033951Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"client":"0","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.088079471Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"client":"1","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.088106455Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"client":"2","":"downstairs","session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.08813416Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.08815961Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"eebb3a95-8ca3-4409-baa5-9e8f2be5e78e is now active with session: b10af835-b8ae-4dc0-94e4-e6446dd83177","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.088188537Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.088212795Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504,"session_id":"b10af835-b8ae-4dc0-94e4-e6446dd83177"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.088238376Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504} {"msg":"Activated sub_volume eebb3a95-8ca3-4409-baa5-9e8f2be5e78e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:28.08826038Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 136 len 4096 data: 8 03/30 Read block 567 len 32768 04/30 Read block 329 len 12288 05/30 Read block 580 len 40960 06/30 Read block 125 len 16384 07/30 Write block 475 len 4096 data: 9 08/30 Read block 260 len 40960 09/30 Write block 175 len 12288 data: 6 8 8 10/30 Read block 279 len 32768 11/30 Write block 177 len 24576 data: 9 5 6 10 8 8 12/30 Read block 560 len 40960 13/30 Write block 4 len 12288 data: 9 10 12 14/30 Read block 225 len 16384 15/30 Read block 133 len 4096 16/30 Write block 371 len 40960 data: 6 6 7 7 4 4 4 6 7 6 17/30 Read block 228 len 24576 18/30 Read block 408 len 20480 19/30 Read block 184 len 28672 20/30 Read block 395 len 32768 21/30 Read block 233 len 8192 22/30 Write block 164 len 16384 data: 10 12 9 10 23/30 Write block 385 len 16384 data: 8 6 5 5 24/30 Write block 473 len 12288 data: 8 9 10 25/30 Read block 149 len 8192 26/30 Write block 84 len 40960 data: 6 5 6 6 6 7 5 7 7 10 27/30 Read block 10 len 40960 28/30 Read block 466 len 32768 29/30 Write block 551 len 40960 data: 10 11 10 12 12 11 9 8 8 7 30/30 Read block 525 len 40960 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 1 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 0 0 26 Done 3 29 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:28.124078615Z","hostname":"bmat-EVT22200007-0000ead9","pid":1504} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 130 130 130 F T T <--- 1 020-039 73 73 73 135 135 135 F F F 2 040-059 74 74 74 136 136 136 F F F 3 060-079 71 71 71 133 133 133 F F F 4 080-099 75 75 75 137 137 137 F T T <--- 5 100-119 77 77 77 145 145 145 F F F 6 120-139 77 77 77 145 145 145 T T T 7 140-159 77 77 77 145 145 145 F F F 8 160-179 76 76 76 143 143 143 F T T <--- 9 180-199 76 76 76 143 143 143 F T T <--- 10 200-219 73 73 73 135 135 135 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 75 75 75 137 137 137 F F F 13 260-279 75 75 75 137 137 137 F F F 14 280-299 74 74 74 136 136 136 F F F 15 300-319 74 74 74 136 136 136 F F F 16 320-339 77 77 77 145 145 145 F F F 17 340-359 75 75 75 137 137 137 F F F 18 360-379 76 76 76 142 142 142 F T T <--- 19 380-399 75 75 75 137 137 137 F T T <--- 20 400-419 76 76 76 142 142 142 F F F 21 420-439 72 72 72 134 134 134 F F F 22 440-459 72 72 72 134 134 134 F F F 23 460-479 74 74 74 136 136 136 F T T <--- 24 480-499 72 72 72 134 134 134 F F F 25 500-519 76 76 76 144 144 144 F F F 26 520-539 75 75 75 137 137 137 F F F 27 540-559 77 77 77 145 145 145 F T T <--- 28 560-579 77 77 77 145 145 145 F T T <--- 29 580-599 77 77 77 145 145 145 F F F Max gen: 77, Max flush: 145 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 146 146 146 F F F 1 020-039 73 73 73 135 135 135 F F F 2 040-059 74 74 74 136 136 136 F F F 3 060-079 71 71 71 133 133 133 F F F 4 080-099 78 78 78 146 146 146 F F F 5 100-119 77 77 77 145 145 145 F F F 6 120-139 78 78 78 146 146 146 F F F 7 140-159 77 77 77 145 145 145 F F F 8 160-179 78 78 78 146 146 146 F F F 9 180-199 78 78 78 146 146 146 F F F 10 200-219 73 73 73 135 135 135 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 75 75 75 137 137 137 F F F 13 260-279 75 75 75 137 137 137 F F F 14 280-299 74 74 74 136 136 136 F F F 15 300-319 74 74 74 136 136 136 F F F 16 320-339 77 77 77 145 145 145 F F F 17 340-359 75 75 75 137 137 137 F F F 18 360-379 78 78 78 146 146 146 F F F 19 380-399 78 78 78 146 146 146 F F F 20 400-419 76 76 76 142 142 142 F F F 21 420-439 72 72 72 134 134 134 F F F 22 440-459 72 72 72 134 134 134 F F F 23 460-479 78 78 78 146 146 146 F F F 24 480-499 72 72 72 134 134 134 F F F 25 500-519 76 76 76 144 144 144 F F F 26 520-539 75 75 75 137 137 137 F F F 27 540-559 78 78 78 146 146 146 F F F 28 560-579 78 78 78 146 146 146 F F F 29 580-599 77 77 77 145 145 145 F F F Max gen: 78, Max flush: 146 Begin loop 40 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.802090659Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:32.829463864Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.829550215Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.858177003Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.858790908Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.858863993Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.858898451Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.858937258Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512} {"msg":"Crucible d9d9fd4f-a79c-4c52-bd39-70516fdb783c has session id: e395ee27-5a6a-4c16-9e42-5e049515cfa3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.860376688Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"Upstairs opts: Upstairs UUID: d9d9fd4f-a79c-4c52-bd39-70516fdb783c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.860510513Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"Crucible stats registered with UUID: d9d9fd4f-a79c-4c52-bd39-70516fdb783c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.86056458Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:32.860607284Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.860869343Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.861081693Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.861130009Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.861557255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.861617324Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512} {"msg":"d9d9fd4f-a79c-4c52-bd39-70516fdb783c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.861728955Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.861760748Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"client":"0","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.861791318Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"client":"1","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.861821238Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"client":"2","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.861850285Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.861999591Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:58989)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.862037115Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.862067485Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:37629)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.862510863Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"ds_connection connected from Ok(127.0.0.1:54983)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.862608426Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.864404542Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"client":"1","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.865172112Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"client":"0","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.865296188Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.865351216Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"client":"2","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.8663933Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"[0]R flush_numbers[0..12]: [146, 135, 136, 133, 146, 145, 146, 145, 146, 146, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.866503428Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"downstairs","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"[0]R generation[0..12]: [78, 73, 74, 71, 78, 77, 78, 77, 78, 78, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.866562505Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"downstairs","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.866622403Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"downstairs","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"[1]R flush_numbers[0..12]: [146, 135, 136, 133, 146, 145, 146, 145, 146, 146, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.866679597Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"downstairs","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"[1]R generation[0..12]: [78, 73, 74, 71, 78, 77, 78, 77, 78, 78, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.866744585Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"downstairs","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.866804163Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"downstairs","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"[2]R flush_numbers[0..12]: [146, 135, 136, 133, 146, 145, 146, 145, 146, 146, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.866860976Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"downstairs","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"[2]R generation[0..12]: [78, 73, 74, 71, 78, 77, 78, 77, 78, 78, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.866916666Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"downstairs","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.866975933Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"downstairs","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"Max found gen is 79","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.867024048Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"Generation requested: 80 >= found:79","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.867091272Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"downstairs","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.867138996Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"downstairs","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.867186721Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"":"downstairs","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.86723716Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.867286268Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"client":"0","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.867341998Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"client":"1","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.867397277Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"client":"2","":"downstairs","session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.8674543Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"Set Downstairs and Upstairs active after reconciliationDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.867514399Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"d9d9fd4f-a79c-4c52-bd39-70516fdb783c is now active with session: e395ee27-5a6a-4c16-9e42-5e049515cfa3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.86758017Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.867630289Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512,"session_id":"e395ee27-5a6a-4c16-9e42-5e049515cfa3"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.867683284Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512} {"msg":"Activated sub_volume d9d9fd4f-a79c-4c52-bd39-70516fdb783c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:32.867728353Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 492 len 4096 data: 4 04/30 Read block 365 len 32768 05/30 Read block 161 len 8192 06/30 Flush 07/30 Write block 110 len 4096 data: 4 08/30 Read block 96 len 12288 09/30 Write block 25 len 8192 data: 3 3 10/30 Write block 256 len 20480 data: 5 7 6 7 10 11/30 Flush 12/30 Write block 543 len 20480 data: 8 5 6 5 6 13/30 Write block 240 len 36864 data: 10 10 8 7 6 7 8 8 8 14/30 Read block 467 len 20480 15/30 Read block 44 len 8192 16/30 Read block 27 len 16384 17/30 Read block 6 len 16384 18/30 Read block 202 len 40960 19/30 Write block 171 len 12288 data: 8 7 7 20/30 Read block 284 len 36864 21/30 Write block 96 len 20480 data: 8 9 7 7 6 22/30 Read block 419 len 16384 23/30 Write block 355 len 16384 data: 5 8 10 12 24/30 Read block 391 len 8192 25/30 Read block 350 len 40960 26/30 Write block 50 len 32768 data: 5 7 7 7 6 5 6 5 27/30 Write block 259 len 16384 data: 8 11 11 10 28/30 Read block 51 len 20480 29/30 Write block 64 len 32768 data: 7 5 4 3 6 7 7 4 30/30 Write block 555 len 4096 data: 13 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 1 Sent Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Read 2 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 2 2 32 Done 0 26 26 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:32.901787686Z","hostname":"bmat-EVT22200007-0000ead9","pid":1512} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 146 146 146 F F F 1 020-039 73 80 80 135 150 150 F F F <--- 2 040-059 74 74 74 136 136 136 F T T <--- 3 060-079 71 71 71 133 133 133 F T T <--- 4 080-099 78 78 78 146 146 146 F T T <--- 5 100-119 77 80 80 145 150 150 F T T <--- 6 120-139 78 78 78 146 146 146 F F F 7 140-159 77 77 77 145 145 145 F F F 8 160-179 78 78 78 146 146 146 F T T <--- 9 180-199 78 78 78 146 146 146 F F F 10 200-219 73 73 73 135 135 135 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 75 80 80 137 150 150 F T T <--- 13 260-279 75 80 80 137 150 150 F T T <--- 14 280-299 74 74 74 136 136 136 F F F 15 300-319 74 74 74 136 136 136 F F F 16 320-339 77 77 77 145 145 145 F F F 17 340-359 75 75 75 137 137 137 F T T <--- 18 360-379 78 78 78 146 146 146 F F F 19 380-399 78 78 78 146 146 146 F F F 20 400-419 76 76 76 142 142 142 F F F 21 420-439 72 72 72 134 134 134 F F F 22 440-459 72 72 72 134 134 134 F F F 23 460-479 78 78 78 146 146 146 F F F 24 480-499 72 80 80 134 149 149 F F F <--- 25 500-519 76 76 76 144 144 144 F F F 26 520-539 75 75 75 137 137 137 F F F 27 540-559 78 78 78 146 146 146 F T T <--- 28 560-579 78 78 78 146 146 146 F F F 29 580-599 77 77 77 145 145 145 F F F Max gen: 80, Max flush: 150 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 146 146 146 F F F 1 020-039 80 80 80 150 150 150 F F F 2 040-059 81 81 81 151 151 151 F F F 3 060-079 81 81 81 151 151 151 F F F 4 080-099 81 81 81 151 151 151 F F F 5 100-119 81 81 81 151 151 151 F F F 6 120-139 78 78 78 146 146 146 F F F 7 140-159 77 77 77 145 145 145 F F F 8 160-179 81 81 81 151 151 151 F F F 9 180-199 78 78 78 146 146 146 F F F 10 200-219 73 73 73 135 135 135 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 81 81 81 151 151 151 F F F 13 260-279 81 81 81 151 151 151 F F F 14 280-299 74 74 74 136 136 136 F F F 15 300-319 74 74 74 136 136 136 F F F 16 320-339 77 77 77 145 145 145 F F F 17 340-359 81 81 81 151 151 151 F F F 18 360-379 78 78 78 146 146 146 F F F 19 380-399 78 78 78 146 146 146 F F F 20 400-419 76 76 76 142 142 142 F F F 21 420-439 72 72 72 134 134 134 F F F 22 440-459 72 72 72 134 134 134 F F F 23 460-479 78 78 78 146 146 146 F F F 24 480-499 80 80 80 149 149 149 F F F 25 500-519 76 76 76 144 144 144 F F F 26 520-539 75 75 75 137 137 137 F F F 27 540-559 81 81 81 151 151 151 F F F 28 560-579 78 78 78 146 146 146 F F F 29 580-599 77 77 77 145 145 145 F F F Max gen: 81, Max flush: 151 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.564835113Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.59355793Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.594164952Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.594248327Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.594276894Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.594315991Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520} {"msg":"Crucible 52934f3f-4ec4-4cd9-8534-1581ae600b17 has session id: 72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.595204671Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"Upstairs opts: Upstairs UUID: 52934f3f-4ec4-4cd9-8534-1581ae600b17, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.595346852Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"Crucible stats registered with UUID: 52934f3f-4ec4-4cd9-8534-1581ae600b17","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.595385489Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:37.595413755Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.595813957Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.595873164Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.595913314Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.596226394Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.59626493Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520} {"msg":"52934f3f-4ec4-4cd9-8534-1581ae600b17 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.596523011Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.596560385Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"client":"0","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.596595234Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"client":"1","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.596625564Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"client":"2","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.596654652Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.596722937Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.596776473Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"ds_connection connected from Ok(127.0.0.1:58892)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.59695197Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"ds_connection connected from Ok(127.0.0.1:34125)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.596993733Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"ds_connection connected from Ok(127.0.0.1:48721)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.59702769Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.599317454Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"client":"0","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.599372002Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"client":"1","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.599407512Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"client":"2","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.600227435Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.600324888Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"[0]R flush_numbers[0..12]: [146, 150, 151, 151, 151, 151, 146, 145, 151, 146, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.600710341Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"downstairs","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"[0]R generation[0..12]: [78, 80, 81, 81, 81, 81, 78, 77, 81, 78, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.600777424Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"downstairs","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.600841581Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"downstairs","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"[1]R flush_numbers[0..12]: [146, 150, 151, 151, 151, 151, 146, 145, 151, 146, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.600902191Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"downstairs","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"[1]R generation[0..12]: [78, 80, 81, 81, 81, 81, 78, 77, 81, 78, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.600960757Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601096406Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"downstairs","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"[2]R flush_numbers[0..12]: [146, 150, 151, 151, 151, 151, 146, 145, 151, 146, 135, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601159Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"downstairs","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"[2]R generation[0..12]: [78, 80, 81, 81, 81, 81, 78, 77, 81, 78, 73, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601216063Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"downstairs","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601278987Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"downstairs","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601330449Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"downstairs","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601384045Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"downstairs","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"Next flush: 152","v":0Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601441759Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"downstairs","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601500576Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"":"downstairs","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601553561Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601601626Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"client":"0","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601655102Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"client":"1","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601708748Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"client":"2","":"downstairs","session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601762995Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601813656Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"52934f3f-4ec4-4cd9-8534-1581ae600b17 is now active with session: 72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601870127Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601918313Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520,"session_id":"72d9a0a7-bd1f-4eec-bc8e-3f3ab20ffcde"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.601968622Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520} {"msg":"Activated sub_volume 52934f3f-4ec4-4cd9-8534-1581ae600b17","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:37.602012069Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 525 len 36864 data: 6 5 5 6 8 7 7 7 9 02/30 Read block 190 len 4096 03/30 Read block 481 len 8192 04/30 Flush 05/30 Write block 133 len 28672 data: 7 8 7 9 9 10 11 06/30 Write block 553 len 4096 data: 11 07/30 Flush 08/30 Write block 430 len 24576 data: 8 7 6 8 8 9 09/30 Write block 567 len 16384 data: 7 8 8 9 10/30 Write block 376 len 12288 data: 5 5 7 11/30 Write block 358 len 12288 data: 13 11 10 12/30 Write block 413 len 32768 data: 4 3 4 4 3 2 2 4 13/30 Write block 162 len 12288 data: 10 12 11 14/30 Read block 22 len 12288 15/30 Write block 200 len 40960 data: 9 9 8 8 6 8 9 10 9 9 16/30 Write block 283 len 24576 data: 6 8 6 8 7 7 17/30 Write block 577 len 36864 data: 6 7 6 7 7 7 7 9 7 18/30 Write block 524 len 40960 data: 6 7 6 6 7 9 8 8 8 10 19/30 Read block 202 len 40960 20/30 Write block 446 len 8192 data: 8 6 21/30 Write block 127 len 36864 data: 7 6 8 8 7 7 8 9 8 22/30 Write block 556 len 32768 data: 12 10 9 9 8 6 7 8 23/30 Read block 103 len 32768 24/30 Write block 270 len 8192 data: 7 7 25/30 Flush 26/30 Write block 441 len 24576 data: 7 7 5 7 8 9 27/30 Read block 221 len 32768 28/30 Read block 155 len 24576 29/30 Read block 562 len 16384 30/30 Write block 364 len 36864 data: 7 8 7 7 7 6 6 7 7 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 2 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 7 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Read 8 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:37.638196519Z","hostname":"bmat-EVT22200007-0000ead9","pid":1520} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 146 146 146 F F F 1 020-039 80 80 80 150 150 150 F F F 2 040-059 81 81 81 151 151 151 F F F 3 060-079 81 81 81 151 151 151 F F F 4 080-099 81 81 81 151 151 151 F F F 5 100-119 81 81 81 151 151 151 F F F 6 120-139 82 82 78 154 154 146 F F F <--- 7 140-159 77 77 77 145 145 145 F F F 8 160-179 82 82 81 154 154 151 F F F <--- 9 180-199 78 78 78 146 146 146 F F F 10 200-219 82 82 73 154 154 135 F F F <--- 11 220-239 73 73 73 135 135 135 F F F 12 240-259 81 81 81 151 151 151 F F F 13 260-279 82 82 81 154 154 151 F F F <--- 14 280-299 82 82 74 154 154 136 F F F <--- 15 300-319 74 74 74 136 136 136 F F F 16 320-339 77 77 77 145 145 145 F F F 17 340-359 82 82 81 154 154 151 F F F <--- 18 360-379 82 82 78 154 154 146 T T F <--- 19 380-399 78 78 78 146 146 146 F F F 20 400-419 82 82 76 154 154 142 F F F <--- 21 420-439 82 82 72 154 154 134 F F F <--- 22 440-459 82 82 72 154 154 134 T T F <--- 23 460-479 78 78 78 146 146 146 F F F 24 480-499 80 80 80 149 149 149 F F F 25 500-519 76 76 76 144 144 144 F F F 26 520-539 82 82 75 154 154 137 F F T <--- 27 540-559 82 82 81 154 154 151 F F F <--- 28 560-579 82 82 78 154 154 146 F F F <--- 29 580-599 82 82 77 154 154 145 F F F <--- Max gen: 82, Max flush: 154 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 146 146 146 F F F 1 020-039 80 80 80 150 150 150 F F F 2 040-059 81 81 81 151 151 151 F F F 3 060-079 81 81 81 151 151 151 F F F 4 080-099 81 81 81 151 151 151 F F F 5 100-119 81 81 81 151 151 151 F F F 6 120-139 82 82 82 154 154 154 F F F 7 140-159 77 77 77 145 145 145 F F F 8 160-179 82 82 82 154 154 154 F F F 9 180-199 78 78 78 146 146 146 F F F 10 200-219 82 82 82 154 154 154 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 81 81 81 151 151 151 F F F 13 260-279 82 82 82 154 154 154 F F F 14 280-299 82 82 82 154 154 154 F F F 15 300-319 74 74 74 136 136 136 F F F 16 320-339 77 77 77 145 145 145 F F F 17 340-359 82 82 82 154 154 154 F F F 18 360-379 83 83 83 155 155 155 F F F 19 380-399 78 78 78 146 146 146 F F F 20 400-419 82 82 82 154 154 154 F F F 21 420-439 82 82 82 154 154 154 F F F 22 440-459 83 83 83 155 155 155 F F F 23 460-479 78 78 78 146 146 146 F F F 24 480-499 80 80 80 149 149 149 F F F 25 500-519 76 76 76 144 144 144 F F F 26 520-539 82 82 82 154 154 154 F F F 27 540-559 82 82 82 154 154 154 F F F 28 560-579 82 82 82 154 154 154 F F F 29 580-599 82 82 82 154 154 154 F F F Max gen: 83, Max flush: 155 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.292669019Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.318935583Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.319552284Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.319693093Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.319771338Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.319824593Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528} {"msg":"Crucible 6d8b9509-b83b-4755-80c1-2a0d420abe11 has session id: f73d6399-0020-4368-bf8c-a4a35a5f1f4a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.320902948Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"Upstairs opts: Upstairs UUID: 6d8b9509-b83b-4755-80c1-2a0d420abe11, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.320985842Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"Crucible stats registered with UUID: 6d8b9509-b83b-4755-80c1-2a0d420abe11","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.321065159Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:42.321100669Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.321644496Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.321716589Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.321755867Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.322026222Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.322067945Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528} {"msg":"6d8b9509-b83b-4755-80c1-2a0d420abe11 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.322229895Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.322268001Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"client":"0","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.322306036Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"client":"1","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.322342959Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"client":"2","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.322380253Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.322424721Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.322463187Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53361)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.32285874Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51085)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.322912667Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:42.322963267Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:42.323012264Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:42.323061522Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.323266477Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.323308591Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.324866796Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"client":"0","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.325001933Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"client":"1","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.32564185Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:42.325867257Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.325237123Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43891)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.325698747Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.328258124Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"client":"2","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"[0]R flush_numbers[0..12]: [146, 150, 151, 151, 151, 151, 154, 145, 154, 146, 154, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329336288Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"downstairs","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"[0]R generation[0..12]: [78, 80, 81, 81, 81, 81, 82, 77, 82, 78, 82, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329421647Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"downstairs","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329460695Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"downstairs","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"[1]R flush_numbers[0..12]: [146, 150, 151, 151, 151, 151, 154, 145, 154, 146, 154, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329517638Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"downstairs","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"[1]R generation[0..12]: [78, 80, 81, 81, 81, 81, 82, 77, 82, 78, 82, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329551064Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"downstairs","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329586253Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"downstairs","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"[2]R flush_numbers[0..12]: [146, 150, 151, 151, 151, 151, 154, 145, 154, 146, 154, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329620271Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"downstairs","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"[2]R generation[0..12]: [78, 80, 81, 81, 81, 81, 82, 77, 82, 78, 82, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329653016Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"downstairs","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329688706Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"downstairs","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329717223Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"downstairs","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329746841Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"downstairs","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329775007Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"downstairs","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329803183Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"":"downstairs","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329842581Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329869705Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"client":"0","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329899323Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"client":"1","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329928511Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"client":"2","":"downstairs","session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329958691Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.329987007Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"6d8b9509-b83b-4755-80c1-2a0d420abe11 is now active with session: f73d6399-0020-4368-bf8c-a4a35a5f1f4a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.33001894Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:11:52.330045653Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528,"session_id":"f73d6399-0020-4368-bf8c-a4a35a5f1f4a"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.33008446Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528} {"msg":"Activated sub_volume 6d8b9509-b83b-4755-80c1-2a0d420abe11","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:52.330108828Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 448 len 12288 data: 7 8 7 02/30 Write block 60 len 24576 data: 5 5 5 5 8 6 03/30 Read block 8 len 36864 04/30 Write block 422 len 32768 data: 6 7 8 8 7 6 6 8 05/30 Read block 331 len 28672 06/30 Read block 506 len 36864 07/30 Read block 539 len 28672 08/30 Read block 261 len 4096 09/30 Read block 576 len 8192 10/30 Read block 8 len 24576 11/30 Read block 250 len 20480 12/30 Read block 425 len 8192 13/30 Read block 375 len 16384 14/30 Write block 484 len 4096 data: 4 15/30 Read block 56 len 4096 16/30 Read block 0 len 8192 17/30 Write block 11 len 4096 data: 6 18/30 Write block 98 len 36864 data: 8 8 7 3 4 3 5 6 5 19/30 Read block 109 len 40960 20/30 Read block 452 len 28672 21/30 Write block 396 len 36864 data: 6 7 6 6 9 11 10 9 11 22/30 Read block 283 len 8192 23/30 Write block 287 len 8192 data: 8 8 24/30 Read block 379 len 28672 25/30 Write block 320 len 16384 data: 11 10 11 11 26/30 Write block 59 len 4096 data: 4 27/30 Write block 577 len 40960 data: 7 8 7 8 8 8 8 10 8 7 28/30 Read block 234 len 4096 29/30 Write block 45 len 20480 data: 6 5 6 4 5 30/30 Write block 270 len 36864 data: 8 8 8 8 8 7 8 6 7 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 9 Done Done Sent false 1003 Acked Write 8 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Read 7 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Write 5 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:52.368700354Z","hostname":"bmat-EVT22200007-0000ead9","pid":1528} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 146 146 146 T T F <--- 1 020-039 80 80 80 150 150 150 F F F 2 040-059 81 81 81 151 151 151 T T F <--- 3 060-079 81 81 81 151 151 151 T T T 4 080-099 81 81 81 151 151 151 T T F <--- 5 100-119 81 81 81 151 151 151 T T F <--- 6 120-139 82 82 82 154 154 154 F F F 7 140-159 77 77 77 145 145 145 F F F 8 160-179 82 82 82 154 154 154 F F F 9 180-199 78 78 78 146 146 146 F F F 10 200-219 82 82 82 154 154 154 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 81 81 81 151 151 151 F F F 13 260-279 82 82 82 154 154 154 T T F <--- 14 280-299 82 82 82 154 154 154 T T F <--- 15 300-319 74 74 74 136 136 136 F F F 16 320-339 77 77 77 145 145 145 T T F <--- 17 340-359 82 82 82 154 154 154 F F F 18 360-379 83 83 83 155 155 155 F F F 19 380-399 78 78 78 146 146 146 T T F <--- 20 400-419 82 82 82 154 154 154 T T F <--- 21 420-439 82 82 82 154 154 154 T T F <--- 22 440-459 83 83 83 155 155 155 T T T 23 460-479 78 78 78 146 146 146 F F F 24 480-499 80 80 80 149 149 149 T T F <--- 25 500-519 76 76 76 144 144 144 F F F 26 520-539 82 82 82 154 154 154 F F F 27 540-559 82 82 82 154 154 154 F F F 28 560-579 82 82 82 154 154 154 T T F <--- 29 580-599 82 82 82 154 154 154 T T F <--- Max gen: 83, Max flush: 155 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 156 156 156 F F F 1 020-039 80 80 80 150 150 150 F F F 2 040-059 84 84 84 156 156 156 F F F 3 060-079 84 84 84 156 156 156 F F F 4 080-099 84 84 84 156 156 156 F F F 5 100-119 84 84 84 156 156 156 F F F 6 120-139 82 82 82 154 154 154 F F F 7 140-159 77 77 77 145 145 145 F F F 8 160-179 82 82 82 154 154 154 F F F 9 180-199 78 78 78 146 146 146 F F F 10 200-219 82 82 82 154 154 154 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 81 81 81 151 151 151 F F F 13 260-279 84 84 84 156 156 156 F F F 14 280-299 84 84 84 156 156 156 F F F 15 300-319 74 74 74 136 136 136 F F F 16 320-339 84 84 84 156 156 156 F F F 17 340-359 82 82 82 154 154 154 F F F 18 360-379 83 83 83 155 155 155 F F F 19 380-399 84 84 84 156 156 156 F F F 20 400-419 84 84 84 156 156 156 F F F 21 420-439 84 84 84 156 156 156 F F F 22 440-459 84 84 84 156 156 156 F F F 23 460-479 78 78 78 146 146 146 F F F 24 480-499 84 84 84 156 156 156 F F F 25 500-519 76 76 76 144 144 144 F F F 26 520-539 82 82 82 154 154 154 F F F 27 540-559 82 82 82 154 154 154 F F F 28 560-579 84 84 84 156 156 156 F F F 29 580-599 84 84 84 156 156 156 F F F Max gen: 84, Max flush: 156 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.190231758Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.219095735Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.219603941Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.219692727Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.21971978Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.219758637Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536} {"msg":"Crucible 7ad2c369-b450-42a6-bd9b-4b99603fb57b has session id: 759a4b6d-d23d-41b1-aef0-6b3da29a7dd3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.221101416Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"Upstairs opts: Upstairs UUID: 7ad2c369-b450-42a6-bd9b-4b99603fb57b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.221225992Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"Crucible stats registered with UUID: 7ad2c369-b450-42a6-bd9b-4b99603fb57b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.221271863Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:57.221304919Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.221688428Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.221764048Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.221794077Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.221938253Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.221974264Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536} {"msg":"7ad2c369-b450-42a6-bd9b-4b99603fb57b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.22207337Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.222104432Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"client":"0","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.222134491Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"client":"1","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.222165903Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"client":"2","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.222199279Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.222358966Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.222417351Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"ds_connection connected from Ok(127.0.0.1:63130)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.222534052Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43909)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.222578741Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:57.222615724Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:57.222650563Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:11:57.222872412Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.223105554Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.223171514Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.224706373Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"client":"2","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.225083149Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"client":"0","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.225380027Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:11:57.226101255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.223994469Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"ds_connection connected from Ok(127.0.0.1:55078)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.224101942Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.226788461Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"client":"1","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"[0]R flush_numbers[0..12]: [156, 150, 156, 156, 156, 156, 154, 145, 154, 146, 154, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.228028496Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"downstairs","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"[0]R generation[0..12]: [84, 80, 84, 84, 84, 84, 82, 77, 82, 78, 82, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.228128334Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"downstairs","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.228206048Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"downstairs","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"[1]R flush_numbers[0..12]: [156, 150, 156, 156, 156, 156, 154, 145, 154, 146, 154, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.228258562Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"downstairs","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"[1]R generation[0..12]: [84, 80, 84, 84, 84, 84, 82, 77, 82, 78, 82, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.228302148Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"downstairs","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.228365664Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"downstairs","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"[2]R flush_numbers[0..12]: [156, 150, 156, 156, 156, 156, 154, 145, 154, 146, 154, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.228438267Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"downstairs","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"[2]R generation[0..12]: [84, 80, 84, 84, 84, 84, 82, 77, 82, 78, 82, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.228511452Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"downstairs","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.228575148Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"downstairs","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"Max found gen is 85","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.228618975Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"downstairs","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"Generation requested: 86 >= found:85","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.228674094Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"downstairs","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.228735326Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"downstairs","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.228793731Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"":"downstairs","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:12:07.228857648Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.22894488Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"client":"0","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.22898517Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"client":"1","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.229036572Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"client":"2","":"downstairs","session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.229095699Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.229154275Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"7ad2c369-b450-42a6-bd9b-4b99603fb57b is now active with session: 759a4b6d-d23d-41b1-aef0-6b3da29a7dd3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.229193332Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.229230035Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536,"session_id":"759a4b6d-d23d-41b1-aef0-6b3da29a7dd3"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.229279533Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536} {"msg":"Activated sub_volume 7ad2c369-b450-42a6-bd9b-4b99603fb57b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:07.22932827Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 334 len 4096 data: 4 02/30 Read block 452 len 20480 03/30 Flush 04/30 Write block 121 len 40960 data: 8 6 5 5 5 6 8 7 9 9 05/30 Write block 435 len 36864 data: 10 11 13 14 10 9 8 8 6 06/30 Read block 550 len 8192 07/30 Read block 478 len 36864 08/30 Write block 412 len 36864 data: 5 5 4 5 5 4 3 3 5 09/30 Read block 94 len 16384 10/30 Read block 502 len 28672 11/30 Read block 578 len 24576 12/30 Read block 85 len 28672 13/30 Write block 357 len 24576 data: 11 14 12 11 10 12 14/30 Flush 15/30 Write block 560 len 32768 data: 9 7 8 9 9 8 8 8 16/30 Read block 34 len 16384 17/30 Read block 366 len 8192 18/30 Flush 19/30 Write block 55 len 28672 data: 6 7 6 3 5 6 6 20/30 Write block 501 len 16384 data: 4 5 8 8 21/30 Write block 291 len 40960 data: 9 8 5 7 8 7 7 7 5 6 22/30 Write block 295 len 40960 data: 9 8 8 8 6 7 6 6 7 9 23/30 Flush 24/30 Write block 185 len 36864 data: 8 8 9 10 10 9 11 10 13 25/30 Read block 494 len 12288 26/30 Write block 152 len 12288 data: 6 6 6 27/30 Write block 459 len 36864 data: 12 11 7 10 9 8 7 8 7 28/30 Read block 327 len 4096 29/30 Write block 361 len 4096 data: 11 30/30 Read block 20 len 28672 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:07.273767804Z","hostname":"bmat-EVT22200007-0000ead9","pid":1536} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 156 156 156 F F F 1 020-039 80 80 80 150 150 150 F F F 2 040-059 86 84 86 160 156 160 F F F <--- 3 060-079 86 84 86 160 156 160 F F F <--- 4 080-099 84 84 84 156 156 156 F F F 5 100-119 84 84 84 156 156 156 F F F 6 120-139 86 82 86 158 154 158 F F F <--- 7 140-159 77 77 77 145 145 145 T F T <--- 8 160-179 82 82 82 154 154 154 F F F 9 180-199 78 78 78 146 146 146 T F T <--- 10 200-219 82 82 82 154 154 154 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 81 81 81 151 151 151 F F F 13 260-279 84 84 84 156 156 156 F F F 14 280-299 86 84 86 160 156 160 F F F <--- 15 300-319 86 74 86 160 136 160 F F F <--- 16 320-339 86 84 86 157 156 157 F T F <--- 17 340-359 86 82 86 158 154 158 F F F <--- 18 360-379 86 83 86 158 155 158 T F T <--- 19 380-399 84 84 84 156 156 156 F F F 20 400-419 86 84 86 158 156 158 F F F <--- 21 420-439 86 84 86 158 156 158 F F F <--- 22 440-459 86 84 86 158 156 158 T F T <--- 23 460-479 78 78 78 146 146 146 T F T <--- 24 480-499 84 84 84 156 156 156 F F F 25 500-519 86 76 86 160 144 160 F F F <--- 26 520-539 82 82 82 154 154 154 F F F 27 540-559 82 82 82 154 154 154 F F F 28 560-579 86 84 86 159 156 159 F F F <--- 29 580-599 84 84 84 156 156 156 F F F Max gen: 86, Max flush: 160 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 84 84 156 156 156 F F F 1 020-039 80 80 80 150 150 150 F F F 2 040-059 86 86 86 160 160 160 F F F 3 060-079 86 86 86 160 160 160 F F F 4 080-099 84 84 84 156 156 156 F F F 5 100-119 84 84 84 156 156 156 F F F 6 120-139 86 86 86 158 158 158 F F F 7 140-159 87 87 87 161 161 161 F F F 8 160-179 82 82 82 154 154 154 F F F 9 180-199 87 87 87 161 161 161 F F F 10 200-219 82 82 82 154 154 154 F F F 11 220-239 73 73 73 135 135 135 F F F 12 240-259 81 81 81 151 151 151 F F F 13 260-279 84 84 84 156 156 156 F F F 14 280-299 86 86 86 160 160 160 F F F 15 300-319 86 86 86 160 160 160 F F F 16 320-339 86 86 86 157 157 157 F F F 17 340-359 86 86 86 158 158 158 F F F 18 360-379 87 87 87 161 161 161 F F F 19 380-399 84 84 84 156 156 156 F F F 20 400-419 86 86 86 158 158 158 F F F 21 420-439 86 86 86 158 158 158 F F F 22 440-459 87 87 87 161 161 161 F F F 23 460-479 87 87 87 161 161 161 F F F 24 480-499 84 84 84 156 156 156 F F F 25 500-519 86 86 86 160 160 160 F F F 26 520-539 82 82 82 154 154 154 F F F 27 540-559 82 82 82 154 154 154 F F F 28 560-579 86 86 86 159 159 159 F F F 29 580-599 84 84 84 156 156 156 F F F Max gen: 87, Max flush: 161 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:11.984100791Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.019729651Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.020534214Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.020646376Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.02067354Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.020713218Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544} {"msg":"Crucible 12a627da-5b7a-424f-87c5-097ce66d4a8d has session id: 358ad78e-8bd3-43d5-b023-eba9e4d8f1ca","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.021924657Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"Upstairs opts: Upstairs UUID: 12a627da-5b7a-424f-87c5-097ce66d4a8d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.022238769Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"Crucible stats registered with UUID: 12a627da-5b7a-424f-87c5-097ce66d4a8d","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.022297755Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:12.022343516Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.022841151Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.022931881Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.022964876Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.023177718Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.023236243Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544} {"msg":"12a627da-5b7a-424f-87c5-097ce66d4a8d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.023335861Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.023366111Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"client":"0","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.023393615Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"client":"1","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.023420989Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"client":"2","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.023448033Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.023475818Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.023705382Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:50129)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.023754409Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:55539)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.023784589Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:63742)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.024246093Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.026210813Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"client":"0","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.026473272Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"client":"2","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.02677028Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"client":"1","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.027111907Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.02722983Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"[0]R flush_numbers[0..12]: [156, 150, 160, 160, 156, 156, 158, 161, 154, 161, 154, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.027562719Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"downstairs","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"[0]R generation[0..12]: [84, 80, 86, 86, 84, 84, 86, 87, 82, 87, 82, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.027616005Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"downstairs","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.027653289Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"downstairs","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"[1]R flush_numbers[0..12]: [156, 150, 160, 160, 156, 156, 158, 161, 154, 161, 154, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.027687476Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"downstairs","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"[1]R generation[0..12]: [84, 80, 86, 86, 84, 84, 86, 87, 82, 87, 82, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.027725752Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"downstairs","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.027761463Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"downstairs","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"[2]R flush_numbers[0..12]: [156, 150, 160, 160, 156, 156, 158, 161, 154, 161, 154, 135]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.027804939Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"downstairs","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"[2]R generation[0..12]: [84, 80, 86, 86, 84, 84, 86, 87, 82, 87, 82, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.027838175Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"downstairs","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.027874226Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.02791684Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"downstairs","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.02794696Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"downstairs","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.027975216Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"downstairs","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.028003612Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"":"downstairs","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.028034052Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.028061377Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"client":"0","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.028091516Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"client":"1","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.028121355Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"client":"2","":"downstairs","session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.028151465Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.028179621Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"12a627da-5b7a-424f-87c5-097ce66d4a8d is now active with session: 358ad78e-8bd3-43d5-b023-eba9e4d8f1ca","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.028211654Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.028238387Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544,"session_id":"358ad78e-8bd3-43d5-b023-eba9e4d8f1ca"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.028266763Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544} {"msg":"Activated sub_volume 12a627da-5b7a-424f-87c5-097ce66d4a8d","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:12.028295159Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 16 len 40960 data: 4 4 4 5 6 7 8 6 6 4 02/30 Read block 500 len 20480 03/30 Write block 96 len 28672 data: 9 10 9 9 8 4 5 04/30 Read block 233 len 28672 05/30 Read block 169 len 4096 06/30 Read block 484 len 16384 07/30 Write block 396 len 28672 data: 7 8 7 7 10 12 11 08/30 Write block 266 len 8192 data: 7 6 09/30 Flush 10/30 Write block 325 len 40960 data: 7 6 6 5 4 5 4 4 4 5 11/30 Write block 447 len 24576 data: 7 8 9 8 6 5 12/30 Write block 459 len 12288 data: 13 12 8 13/30 Read block 442 len 16384 14/30 Read block 131 len 36864 15/30 Write block 219 len 8192 data: 5 5 16/30 Read block 503 len 16384 17/30 Write block 85 len 28672 data: 6 7 7 7 8 6 8 18/30 Read block 397 len 12288 19/30 Read block 448 len 28672 20/30 Read block 253 len 16384 21/30 Write block 290 len 12288 data: 7 10 9 22/30 Write block 27 len 16384 data: 4 4 4 4 23/30 Write block 256 len 36864 data: 6 8 7 9 12 12 11 11 9 24/30 Flush 25/30 Flush 26/30 Write block 454 len 28672 data: 8 7 9 12 13 14 13 27/30 Read block 583 len 28672 28/30 Read block 325 len 36864 29/30 Write block 205 len 40960 data: 9 10 11 10 10 9 8 7 4 3 30/30 Read block 563 len 32768 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 7 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Write 6 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 0 29 Done 2 29 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:12.070326065Z","hostname":"bmat-EVT22200007-0000ead9","pid":1544} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 84 88 88 156 162 162 T F F <--- 1 020-039 80 88 88 150 163 163 T F F <--- 2 040-059 86 86 86 160 160 160 F F F 3 060-079 86 86 86 160 160 160 F F F 4 080-099 84 88 88 156 163 163 F F F <--- 5 100-119 84 88 88 156 162 162 F F F <--- 6 120-139 86 86 86 158 158 158 F F F 7 140-159 87 87 87 161 161 161 F F F 8 160-179 82 82 82 154 154 154 F F F 9 180-199 87 87 87 161 161 161 F F F 10 200-219 82 88 88 154 163 163 F T T <--- 11 220-239 73 88 88 135 163 163 F F F <--- 12 240-259 81 88 88 151 163 163 F F F <--- 13 260-279 84 88 88 156 163 163 F F F <--- 14 280-299 86 88 88 160 163 163 F F F <--- 15 300-319 86 86 86 160 160 160 F F F 16 320-339 86 88 88 157 163 163 F F F <--- 17 340-359 86 86 86 158 158 158 F F F 18 360-379 87 87 87 161 161 161 F F F 19 380-399 84 88 88 156 162 162 F F F <--- 20 400-419 86 88 88 158 162 162 F F F <--- 21 420-439 86 86 86 158 158 158 F F F 22 440-459 87 88 88 161 163 163 F T T <--- 23 460-479 87 88 88 161 163 163 F T T <--- 24 480-499 84 84 84 156 156 156 F F F 25 500-519 86 86 86 160 160 160 F F F 26 520-539 82 82 82 154 154 154 F F F 27 540-559 82 82 82 154 154 154 F F F 28 560-579 86 86 86 159 159 159 F F F 29 580-599 84 84 84 156 156 156 F F F Max gen: 88, Max flush: 163 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 162 162 162 F F F 1 020-039 88 88 88 163 163 163 F F F 2 040-059 86 86 86 160 160 160 F F F 3 060-079 86 86 86 160 160 160 F F F 4 080-099 88 88 88 163 163 163 F F F 5 100-119 88 88 88 162 162 162 F F F 6 120-139 86 86 86 158 158 158 F F F 7 140-159 87 87 87 161 161 161 F F F 8 160-179 82 82 82 154 154 154 F F F 9 180-199 87 87 87 161 161 161 F F F 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 88 88 88 163 163 163 F F F 13 260-279 88 88 88 163 163 163 F F F 14 280-299 88 88 88 163 163 163 F F F 15 300-319 86 86 86 160 160 160 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 86 86 86 158 158 158 F F F 18 360-379 87 87 87 161 161 161 F F F 19 380-399 88 88 88 162 162 162 F F F 20 400-419 88 88 88 162 162 162 F F F 21 420-439 86 86 86 158 158 158 F F F 22 440-459 89 89 89 164 164 164 F F F 23 460-479 89 89 89 164 164 164 F F F 24 480-499 84 84 84 156 156 156 F F F 25 500-519 86 86 86 160 160 160 F F F 26 520-539 82 82 82 154 154 154 F F F 27 540-559 82 82 82 154 154 154 F F F 28 560-579 86 86 86 159 159 159 F F F 29 580-599 84 84 84 156 156 156 F F F Max gen: 89, Max flush: 164 Begin loop 45 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.789632999Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.828060039Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.828669506Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.828753281Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.828780946Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.828819773Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552} {"msg":"Crucible 418fc21d-0e25-45f0-b04c-4b9d9edec12a has session id: 07973c8e-bd79-484d-ba40-1afc0719eb51","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.829911685Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"Upstairs opts: Upstairs UUID: 418fc21d-0e25-45f0-b04c-4b9d9edec12a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.829979108Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"Crucible stats registered with UUID: 418fc21d-0e25-45f0-b04c-4b9d9edec12a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.830010731Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:16.830038296Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.830629055Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.830686729Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.830718432Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.831094366Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.831144987Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552} {"msg":"418fc21d-0e25-45f0-b04c-4b9d9edec12a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.831464409Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.831563475Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"client":"0","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.831611821Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"client":"1","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.831660507Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"client":"2","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.831705857Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.831738321Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.831774944Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"ds_connection connected from Ok(127.0.0.1:65063)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.831807298Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"ds_connection connected from Ok(127.0.0.1:46331)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.831839412Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"ds_connection connected from Ok(127.0.0.1:50541)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.832103955Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.833611841Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"client":"2","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.833847788Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"client":"0","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.834230816Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"client":"1","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.834869912Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.834981192Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"[0]R flush_numbers[0..12]: [162, 163, 160, 160, 163, 162, 158, 161, 154, 161, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.83507125Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"downstairs","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"[0]R generation[0..12]: [88, 88, 86, 86, 88, 88, 86, 87, 82, 87, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.835138072Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"downstairs","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.835201999Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"downstairs","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"[1]R flush_numbers[0..12]: [162, 163, 160, 160, 163, 162, 158, 161, 154, 161, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.835265124Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"downstairs","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"[1]R generation[0..12]: [88, 88, 86, 86, 88, 88, 86, 87, 82, 87, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.835325073Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"downstairs","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.835388188Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"downstairs","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"[2]R flush_numbers[0..12]: [162, 163, 160, 160, 163, 162, 158, 161, 154, 161, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.835447144Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"downstairs","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"[2]R generation[0..12]: [88, 88, 86, 86, 88, 88, 86, 87, 82, 87, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.83550542Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"downstairs","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {Wait for a query_work_queue command to finish before sending IO "msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.835575569Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"downstairs","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.835630798Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"downstairs","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.835683041Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"downstairs","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.835737429Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"downstairs","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.835789673Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"":"downstairs","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-04T01:12:16.835852056Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.835910612Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"client":"0","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.835965982Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"client":"1","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.836022473Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"client":"2","":"downstairs","session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.836075749Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.836129976Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"418fc21d-0e25-45f0-b04c-4b9d9edec12a is now active with session: 07973c8e-bd79-484d-ba40-1afc0719eb51","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.836185777Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.836231227Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552,"session_id":"07973c8e-bd79-484d-ba40-1afc0719eb51"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.836280083Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552} {"msg":"Activated sub_volume 418fc21d-0e25-45f0-b04c-4b9d9edec12a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:16.836322688Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 8192 data: 7 6 02/30 Read block 131 len 24576 03/30 Read block 88 len 16384 04/30 Flush 05/30 Write block 304 len 36864 data: 10 9 10 11 11 11 10 10 6 06/30 Write block 390 len 16384 data: 5 7 8 6 07/30 Read block 114 len 4096 08/30 Flush 09/30 Write block 257 len 40960 data: 9 8 10 13 13 12 12 10 7 8 10/30 Write block 535 len 16384 data: 9 11 9 8 11/30 Read block 106 len 12288 12/30 Read block 36 len 40960 13/30 Read block 468 len 4096 14/30 Read block 403 len 32768 15/30 Write block 557 len 32768 data: 11 10 10 10 8 9 10 10 16/30 Write block 262 len 8192 data: 13 13 17/30 Read block 519 len 16384 18/30 Write block 284 len 12288 data: 9 7 9 19/30 Read block 246 len 4096 20/30 Read block 34 len 4096 21/30 Write block 298 len 32768 data: 9 7 8 7 7 8 11 10 22/30 Read block 11 len 36864 23/30 Read block 293 len 36864 24/30 Read block 140 len 16384 25/30 Flush 26/30 Flush 27/30 Write block 370 len 4096 data: 7 28/30 Write block 177 len 16384 data: 10 6 7 11 29/30 Write block 184 len 20480 data: 8 9 9 10 11 30/30 Write block 480 len 36864 data: 8 7 7 7 5 4 4 5 3 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Sent Done false 1001 Acked Read 6 Done Sent Done false 1002 Acked Read 4 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Write 5 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:16.871958883Z","hostname":"bmat-EVT22200007-0000ead9","pid":1552} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 162 162 162 F F F 1 020-039 88 88 88 163 163 163 F F F 2 040-059 86 86 86 160 160 160 F F F 3 060-079 86 86 86 160 160 160 F F F 4 080-099 88 88 88 163 163 163 F F F 5 100-119 88 88 88 162 162 162 F F F 6 120-139 86 86 86 158 158 158 F F F 7 140-159 87 87 87 161 161 161 F F F 8 160-179 82 82 82 154 154 154 T F T <--- 9 180-199 87 87 87 161 161 161 T F T <--- 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 90 88 90 167 163 167 F F F <--- 13 260-279 90 88 90 167 163 167 F F F <--- 14 280-299 90 88 90 167 163 167 F F F <--- 15 300-319 90 86 90 167 160 167 F F F <--- 16 320-339 88 88 88 163 163 163 F F F 17 340-359 90 86 90 165 158 165 F F F <--- 18 360-379 87 87 87 161 161 161 T F T <--- 19 380-399 90 88 90 166 162 166 F F F <--- 20 400-419 88 88 88 162 162 162 F F F 21 420-439 86 86 86 158 158 158 F F F 22 440-459 89 89 89 164 164 164 F F F 23 460-479 89 89 89 164 164 164 F F F 24 480-499 84 84 84 156 156 156 T F T <--- 25 500-519 86 86 86 160 160 160 F F F 26 520-539 90 82 90 167 154 167 F F F <--- 27 540-559 90 82 90 167 154 167 F F F <--- 28 560-579 90 86 90 167 159 167 F F F <--- 29 580-599 84 84 84 156 156 156 F F F Max gen: 90, Max flush: 167 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 162 162 162 F F F 1 020-039 88 88 88 163 163 163 F F F 2 040-059 86 86 86 160 160 160 F F F 3 060-079 86 86 86 160 160 160 F F F 4 080-099 88 88 88 163 163 163 F F F 5 100-119 88 88 88 162 162 162 F F F 6 120-139 86 86 86 158 158 158 F F F 7 140-159 87 87 87 161 161 161 F F F 8 160-179 91 91 91 168 168 168 F F F 9 180-199 91 91 91 168 168 168 F F F 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 90 90 90 167 167 167 F F F 13 260-279 90 90 90 167 167 167 F F F 14 280-299 90 90 90 167 167 167 F F F 15 300-319 90 90 90 167 167 167 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 90 90 90 165 165 165 F F F 18 360-379 91 91 91 168 168 168 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 88 88 88 162 162 162 F F F 21 420-439 86 86 86 158 158 158 F F F 22 440-459 89 89 89 164 164 164 F F F 23 460-479 89 89 89 164 164 164 F F F 24 480-499 91 91 91 168 168 168 F F F 25 500-519 86 86 86 160 160 160 F F F 26 520-539 90 90 90 167 167 167 F F F 27 540-559 90 90 90 167 167 167 F F F 28 560-579 90 90 90 167 167 167 F F F 29 580-599 84 84 84 156 156 156 F F F Max gen: 91, Max flush: 168 Begin loop 46 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.603296966Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.635595232Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.636304186Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.636428522Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.636469453Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.636530925Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560} {"msg":"Crucible e84eedb7-2dcd-4b92-8897-ce05a8e8467f has session id: 1063fef8-1adb-4601-8249-fa2f0d4b8844","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.637567647Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"Upstairs opts: Upstairs UUID: e84eedb7-2dcd-4b92-8897-ce05a8e8467f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.637701823Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"Crucible stats registered with UUID: e84eedb7-2dcd-4b92-8897-ce05a8e8467f","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.637746942Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:21.637792603Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.638123708Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.638226462Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.638277553Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.638523219Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.638569862Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560} {"msg":"e84eedb7-2dcd-4b92-8897-ce05a8e8467f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.638768244Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.638812672Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"client":"0","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.638851008Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"client":"1","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.638890817Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"client":"2","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.638927149Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.638967789Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.63901837Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"ds_connection connected from Ok(127.0.0.1:60564)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.6391345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"ds_connection connected from Ok(127.0.0.1:63155)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.639186923Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"ds_connection connected from Ok(127.0.0.1:46036)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.639510845Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.641111995Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"client":"2","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.641351679Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"client":"0","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.641771861Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"client":"1","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.641946417Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.642096444Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"[0]R flush_numbers[0..12]: [162, 163, 160, 160, 163, 162, 158, 161, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.642573659Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"downstairs","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"[0]R generation[0..12]: [88, 88, 86, 86, 88, 88, 86, 87, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.642617896Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"downstairs","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.642654218Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"downstairs","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"[1]R flush_numbers[0..12]: [162, 163, 160, 160, 163, 162, 158, 161, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.642689167Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"downstairs","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"[1]R generation[0..12]: [88, 88, 86, 86, 88, 88, 86, 87, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.642721792Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"downstairs","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.642756761Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"downstairs","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"[2]R flush_numbers[0..12]: [162, 163, 160, 160, 163, 162, 158, 161, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.642800438Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"downstairs","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"[2]R generation[0..12]: [88, 88, 86, 86, 88, 88, 86, 87, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.642833323Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"downstairs","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.642868813Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"downstairs","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.64289755Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"downstairs","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.642927189Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"downstairs","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.642955755Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"downstairs","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.642983991Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"":"downstairs","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.643015493Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"Transition from WaitQuorum to Active","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2026-03-04T01:12:21.643062737Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"client":"0","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.643094169Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"client":"1","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.643123557Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"client":"2","":"downstairs","session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.643153797Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.643187163Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"e84eedb7-2dcd-4b92-8897-ce05a8e8467f is now active with session: 1063fef8-1adb-4601-8249-fa2f0d4b8844","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.643219447Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.64324609Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560,"session_id":"1063fef8-1adb-4601-8249-fa2f0d4b8844"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.643273975Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560} {"msg":"Activated sub_volume e84eedb7-2dcd-4b92-8897-ce05a8e8467f","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:21.643302632Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 444 len 8192 data: 8 9 02/30 Write block 341 len 36864 data: 8 7 4 4 4 6 5 5 5 03/30 Write block 476 len 4096 data: 8 04/30 Flush 05/30 Write block 29 len 12288 data: 5 5 5 06/30 Read block 114 len 12288 07/30 Write block 110 len 20480 data: 5 5 6 12 11 08/30 Flush 09/30 Write block 28 len 36864 data: 5 6 6 6 6 6 6 6 7 10/30 Write block 364 len 12288 data: 8 9 8 11/30 Read block 277 len 36864 12/30 Read block 209 len 36864 13/30 Write block 134 len 12288 data: 10 9 10 14/30 Read block 190 len 36864 15/30 Read block 501 len 36864 16/30 Flush 17/30 Write block 437 len 24576 data: 14 15 11 10 9 9 18/30 Read block 110 len 4096 19/30 Read block 46 len 4096 20/30 Write block 132 len 40960 data: 8 9 11 10 11 10 11 12 10 9 21/30 Read block 239 len 40960 22/30 Flush 23/30 Write block 519 len 16384 data: 5 4 4 6 24/30 Write block 426 len 24576 data: 8 7 7 9 9 8 25/30 Read block 113 len 36864 26/30 Read block 66 len 8192 27/30 Read block 401 len 20480 28/30 Read block 393 len 20480 29/30 Read block 59 len 36864 30/30 Write block 0 len 20480 data: 2 2 5 8 10 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Write 10 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 4 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 26 1 28 Done 25 0 25 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1003)) Some(JobId(1021)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:21.684249401Z","hostname":"bmat-EVT22200007-0000ead9","pid":1560} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 88 162 162 162 T F T <--- 1 020-039 92 88 92 171 163 171 F F F <--- 2 040-059 86 86 86 160 160 160 F F F 3 060-079 86 86 86 160 160 160 F F F 4 080-099 88 88 88 163 163 163 F F F 5 100-119 92 88 92 170 162 170 F F F <--- 6 120-139 92 86 92 172 158 172 F F F <--- 7 140-159 92 87 92 172 161 172 F F F <--- 8 160-179 91 91 91 168 168 168 F F F 9 180-199 91 91 91 168 168 168 F F F 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 90 90 90 167 167 167 F F F 13 260-279 90 90 90 167 167 167 F F F 14 280-299 90 90 90 167 167 167 F F F 15 300-319 90 90 90 167 167 167 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 92 92 92 169 169 169 F F F 18 360-379 92 91 92 171 168 171 F F F <--- 19 380-399 90 90 90 166 166 166 F F F 20 400-419 88 88 88 162 162 162 F F F 21 420-439 92 86 92 172 158 172 T F T <--- 22 440-459 92 92 92 172 169 172 F F F <--- 23 460-479 92 92 92 169 169 169 F F F 24 480-499 91 91 91 168 168 168 F F F 25 500-519 86 86 86 160 160 160 T F T <--- 26 520-539 90 90 90 167 167 167 T F T <--- 27 540-559 90 90 90 167 167 167 F F F 28 560-579 90 90 90 167 167 167 F F F 29 580-599 84 84 84 156 156 156 F F F Max gen: 92, Max flush: 172 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 173 173 173 F F F 1 020-039 92 92 92 171 171 171 F F F 2 040-059 86 86 86 160 160 160 F F F 3 060-079 86 86 86 160 160 160 F F F 4 080-099 88 88 88 163 163 163 F F F 5 100-119 92 92 92 170 170 170 F F F 6 120-139 92 92 92 172 172 172 F F F 7 140-159 92 92 92 172 172 172 F F F 8 160-179 91 91 91 168 168 168 F F F 9 180-199 91 91 91 168 168 168 F F F 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 90 90 90 167 167 167 F F F 13 260-279 90 90 90 167 167 167 F F F 14 280-299 90 90 90 167 167 167 F F F 15 300-319 90 90 90 167 167 167 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 92 92 92 169 169 169 F F F 18 360-379 92 92 92 171 171 171 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 88 88 88 162 162 162 F F F 21 420-439 93 93 93 173 173 173 F F F 22 440-459 92 92 92 172 172 172 F F F 23 460-479 92 92 92 169 169 169 F F F 24 480-499 91 91 91 168 168 168 F F F 25 500-519 93 93 93 173 173 173 F F F 26 520-539 93 93 93 173 173 173 F F F 27 540-559 90 90 90 167 167 167 F F F 28 560-579 90 90 90 167 167 167 F F F 29 580-599 84 84 84 156 156 156 F F F Max gen: 93, Max flush: 173 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.435542944Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.47058887Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.471712795Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.471793926Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.471820619Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.471859496Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568} {"msg":"Crucible cc2ebcfb-a2a4-427c-a925-fee3b86ef2ee has session id: 3555d960-7a5d-4b42-9e9d-bd989e4a4876","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.473605603Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"Upstairs opts: Upstairs UUID: cc2ebcfb-a2a4-427c-a925-fee3b86ef2ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.473673998Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"Crucible stats registered with UUID: cc2ebcfb-a2a4-427c-a925-fee3b86ef2ee","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.473706042Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:26.473734177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.474312011Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.474391539Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.474445866Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.475020033Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.475102346Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568} {"msg":"cc2ebcfb-a2a4-427c-a925-fee3b86ef2ee active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.475208596Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.475263986Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"client":"0","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.475322402Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"client":"1","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.47537736Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"client":"2","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.4754378Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.475499122Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.475657495Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"ds_connection connected from Ok(127.0.0.1:47194)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.475717865Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"ds_connection connected from Ok(127.0.0.1:37859)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.475782743Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"ds_connection connected from Ok(127.0.0.1:58800)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.476183567Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.477900296Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"client":"1","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.477958381Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"client":"2","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.478121433Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"client":"0","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479162485Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479206602Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"[0]R flush_numbers[0..12]: [173, 171, 160, 160, 163, 170, 172, 172, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479325928Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"downstairs","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"[0]R generation[0..12]: [93, 92, 86, 86, 88, 92, 92, 92, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479368202Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"downstairs","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479404043Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"downstairs","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"[1]R flush_numbers[0..12]: [173, 171, 160, 160, 163, 170, 172, 172, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.47943807Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"downstairs","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"[1]R generation[0..12]: [93, 92, 86, 86, 88, 92, 92, 92, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479470484Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"downstairs","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479505474Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"downstairs","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"[2]R flush_numbers[0..12]: [173, 171, 160, 160, 163, 170, 172, 172, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479539331Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"downstairs","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"[2]R generation[0..12]: [93, 92, 86, 86, 88, 92, 92, 92, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479571554Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"downstairs","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479606233Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"downstairs","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.47963491Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"downstairs","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479664378Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"downstairs","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479692825Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"downstairs","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.47972104Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"":"downstairs","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479750499Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479777412Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"client":"0","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479812221Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"client":"1","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.47984204Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"client":"2","":"downstairs","session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.47987235Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"session_id":"Wait for a query_work_queue command to finish before sending IO 3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.47991773Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"cc2ebcfb-a2a4-427c-a925-fee3b86ef2ee is now active with session: 3555d960-7a5d-4b42-9e9d-bd989e4a4876","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479949493Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.479976636Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568,"session_id":"3555d960-7a5d-4b42-9e9d-bd989e4a4876"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.480005113Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568} {"msg":"Activated sub_volume cc2ebcfb-a2a4-427c-a925-fee3b86ef2ee","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:26.480029631Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 554 len 20480 data: 13 14 13 12 11 03/30 Read block 453 len 36864 04/30 Write block 364 len 28672 data: 9 10 9 8 8 7 8 05/30 Read block 171 len 4096 06/30 Read block 91 len 12288 07/30 Write block 592 len 8192 data: 5 4 08/30 Flush 09/30 Write block 359 len 36864 data: 13 12 12 13 10 10 11 10 9 10/30 Read block 46 len 36864 11/30 Flush 12/30 Write block 92 len 36864 data: 8 11 9 8 10 11 10 10 9 13/30 Write block 570 len 20480 data: 10 8 9 6 7 14/30 Flush 15/30 Write block 494 len 36864 data: 6 8 7 7 6 7 5 5 6 16/30 Read block 471 len 12288 17/30 Write block 287 len 24576 data: 9 9 7 8 11 10 18/30 Read block 221 len 8192 19/30 Read block 555 len 20480 20/30 Flush 21/30 Write block 139 len 8192 data: 13 11 22/30 Read block 340 len 28672 23/30 Write block 404 len 40960 data: 12 11 11 10 10 8 6 6 6 6 24/30 Write block 570 len 28672 data: 11 9 10 7 8 8 6 25/30 Write block 358 len 12288 data: 15 14 13 26/30 Read block 502 len 40960 27/30 Write block 10 len 24576 data: 6 7 5 6 5 5 28/30 Write block 414 len 20480 data: 5 6 6 5 4 29/30 Read block 524 len 12288 30/30 Read block 382 len 20480 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 5 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Write 2 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 0 26 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1000)) Some(JobId(1019)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:26.519480388Z","hostname":"bmat-EVT22200007-0000ead9","pid":1568} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 93 93 93 173 173 173 T F T <--- 1 020-039 92 92 92 171 171 171 F F F 2 040-059 86 86 86 160 160 160 F F F 3 060-079 86 86 86 160 160 160 F F F 4 080-099 94 88 94 177 163 177 F F F <--- 5 100-119 94 92 94 177 170 177 F F F <--- 6 120-139 92 92 92 172 172 172 T F T <--- 7 140-159 92 92 92 172 172 172 T F T <--- 8 160-179 91 91 91 168 168 168 F F F 9 180-199 91 91 91 168 168 168 F F F 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 90 90 90 167 167 167 F F F 13 260-279 90 90 90 167 167 167 F F F 14 280-299 94 90 94 178 167 178 F F F <--- 15 300-319 90 90 90 167 167 167 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 94 92 94 176 169 176 T F T <--- 18 360-379 94 92 94 176 171 176 T T T <--- 19 380-399 90 90 90 166 166 166 F F F 20 400-419 88 88 88 162 162 162 T F T <--- 21 420-439 93 93 93 173 173 173 F F F 22 440-459 92 92 92 172 172 172 F F F 23 460-479 92 92 92 169 169 169 F F F 24 480-499 94 91 94 178 168 178 F F F <--- 25 500-519 94 93 94 178 173 178 F F F <--- 26 520-539 93 93 93 173 173 173 F F F 27 540-559 94 90 94 175 167 175 F T F <--- 28 560-579 94 90 94 177 167 177 T F T <--- 29 580-599 94 84 94 175 156 175 F F F <--- Max gen: 94, Max flush: 178 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 179 179 179 F F F 1 020-039 92 92 92 171 171 171 F F F 2 040-059 86 86 86 160 160 160 F F F 3 060-079 86 86 86 160 160 160 F F F 4 080-099 94 94 94 177 177 177 F F F 5 100-119 94 94 94 177 177 177 F F F 6 120-139 95 95 95 179 179 179 F F F 7 140-159 95 95 95 179 179 179 F F F 8 160-179 91 91 91 168 168 168 F F F 9 180-199 91 91 91 168 168 168 F F F 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 90 90 90 167 167 167 F F F 13 260-279 90 90 90 167 167 167 F F F 14 280-299 94 94 94 178 178 178 F F F 15 300-319 90 90 90 167 167 167 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 95 95 95 179 179 179 F F F 18 360-379 95 95 95 179 179 179 F F F 19 380-399 90 90 90 166 166 166 F F F 20 400-419 95 95 95 179 179 179 F F F 21 420-439 93 93 93 173 173 173 F F F 22 440-459 92 92 92 172 172 172 F F F 23 460-479 92 92 92 169 169 169 F F F 24 480-499 94 94 94 178 178 178 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 93 93 93 173 173 173 F F F 27 540-559 94 94 94 175 175 175 F F F 28 560-579 95 95 95 179 179 179 F F F 29 580-599 94 94 94 175 175 175 F F F Max gen: 95, Max flush: 179 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.231771392Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.258680578Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.259298241Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.259372107Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.259395514Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.259430724Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576} {"msg":"Crucible 2c71194e-76ba-481c-b885-4959c742a846 has session id: 2c729be7-7180-4e20-8d79-d28e422539a2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.260828561Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"Upstairs opts: Upstairs UUID: 2c71194e-76ba-481c-b885-4959c742a846, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.260894802Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"Crucible stats registered with UUID: 2c71194e-76ba-481c-b885-4959c742a846","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.260924401Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:31.26094933Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.261472396Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.261523928Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.261552374Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.261836106Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.261869221Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576} {"msg":"2c71194e-76ba-481c-b885-4959c742a846 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.26201518Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.262043787Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"client":"0","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.262070921Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"client":"1","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.262097463Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"client":"2","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.262123956Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.262151601Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.262180237Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:50936)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.2626313Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:34786)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.26271684Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:41477)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.262748182Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.263978087Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"client":"0","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.264163233Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"client":"2","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.264406415Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"client":"1","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.264881906Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.265001483Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"[0]R flush_numbers[0..12]: [179, 171, 160, 160, 177, 177, 179, 179, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.265591351Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"downstairs","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"[0]R generation[0..12]: [95, 92, 86, 86, 94, 94, 95, 95, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.265694745Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"downstairs","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.265750075Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"downstairs","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"[1]R flush_numbers[0..12]: [179, 171, 160, 160, 177, 177, 179, 179, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.265801667Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"downstairs","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"[1]R generation[0..12]: [95, 92, 86, 86, 94, 94, 95, 95, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-04T01:12:31.265852317Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"downstairs","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.26591906Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"downstairs","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"[2]R flush_numbers[0..12]: [179, 171, 160, 160, 177, 177, 179, 179, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.265970141Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"downstairs","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"[2]R generation[0..12]: [95, 92, 86, 86, 94, 94, 95, 95, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.266028967Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"downstairs","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.266083756Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"downstairs","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.26612661Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"downstairs","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.26617192Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"downstairs","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.266214945Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"downstairs","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.266267529Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"":"downstairs","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.26631316Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.266354872Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"client":"0","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.266424791Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"client":"1","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.266471072Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"client":"2","":"downstairs","session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.266517585Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.266560219Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"2c71194e-76ba-481c-b885-4959c742a846 is now active with session: 2c729be7-7180-4e20-8d79-d28e422539a2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.266608645Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.266649105Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576,"session_id":"2c729be7-7180-4e20-8d79-d28e422539a2"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.266691679Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576} {"msg":"Activated sub_volume 2c71194e-76ba-481c-b885-4959c742a846","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:31.266728001Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 81 len 28672 data: 6 7 5 7 7 8 8 02/30 Read block 53 len 16384 03/30 Read block 297 len 28672 04/30 Read block 458 len 12288 05/30 Read block 112 len 4096 06/30 Read block 17 len 20480 07/30 Write block 117 len 24576 data: 12 11 10 11 9 7 08/30 Read block 33 len 40960 09/30 Write block 70 len 4096 data: 8 10/30 Read block 360 len 4096 11/30 Read block 212 len 20480 12/30 Write block 258 len 24576 data: 9 11 14 14 14 14 13/30 Write block 56 len 24576 data: 8 7 4 6 7 7 14/30 Write block 396 len 24576 data: 8 9 8 8 11 13 15/30 Flush 16/30 Write block 52 len 24576 data: 8 8 7 7 9 8 17/30 Read block 295 len 8192 18/30 Flush 19/30 Write block 108 len 16384 data: 7 6 6 6 20/30 Flush 21/30 Write block 95 len 32768 data: 9 11 12 11 11 10 5 6 22/30 Read block 392 len 8192 23/30 Read block 519 len 36864 24/30 Write block 61 len 4096 data: 8 25/30 Read block 583 len 16384 26/30 Write block 431 len 8192 data: 9 7 27/30 Read block 587 len 36864 28/30 Read block 220 len 24576 29/30 Read block 282 len 20480 30/30 Read block 279 len 32768 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 24 1 25 Done 30 6 29 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:31.306741322Z","hostname":"bmat-EVT22200007-0000ead9","pid":1576} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 179 179 179 F F F 1 020-039 92 92 92 171 171 171 F F F 2 040-059 96 86 96 181 160 181 F F F <--- 3 060-079 96 86 96 180 160 180 T F T <--- 4 080-099 96 94 96 180 177 180 T T T <--- 5 100-119 96 94 96 182 177 182 T F T <--- 6 120-139 96 95 96 180 179 180 F F F <--- 7 140-159 95 95 95 179 179 179 F F F 8 160-179 91 91 91 168 168 168 F F F 9 180-199 91 91 91 168 168 168 F F F 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 96 90 96 180 167 180 F F F <--- 13 260-279 96 90 96 180 167 180 F F F <--- 14 280-299 94 94 94 178 178 178 F F F 15 300-319 90 90 90 167 167 167 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 95 95 95 179 179 179 F F F 18 360-379 95 95 95 179 179 179 F F F 19 380-399 96 90 96 180 166 180 F F F <--- 20 400-419 96 95 96 180 179 180 F F F <--- 21 420-439 93 93 93 173 173 173 T F T <--- 22 440-459 92 92 92 172 172 172 F F F 23 460-479 92 92 92 169 169 169 F F F 24 480-499 94 94 94 178 178 178 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 93 93 93 173 173 173 F F F 27 540-559 94 94 94 175 175 175 F F F 28 560-579 95 95 95 179 179 179 F F F 29 580-599 94 94 94 175 175 175 F F F Max gen: 96, Max flush: 182 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 179 179 179 F F F 1 020-039 92 92 92 171 171 171 F F F 2 040-059 96 96 96 181 181 181 F F F 3 060-079 97 97 97 183 183 183 F F F 4 080-099 97 97 97 183 183 183 F F F 5 100-119 97 97 97 183 183 183 F F F 6 120-139 96 96 96 180 180 180 F F F 7 140-159 95 95 95 179 179 179 F F F 8 160-179 91 91 91 168 168 168 F F F 9 180-199 91 91 91 168 168 168 F F F 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 96 96 96 180 180 180 F F F 13 260-279 96 96 96 180 180 180 F F F 14 280-299 94 94 94 178 178 178 F F F 15 300-319 90 90 90 167 167 167 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 95 95 95 179 179 179 F F F 18 360-379 95 95 95 179 179 179 F F F 19 380-399 96 96 96 180 180 180 F F F 20 400-419 96 96 96 180 180 180 F F F 21 420-439 97 97 97 183 183 183 F F F 22 440-459 92 92 92 172 172 172 F F F 23 460-479 92 92 92 169 169 169 F F F 24 480-499 94 94 94 178 178 178 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 93 93 93 173 173 173 F F F 27 540-559 94 94 94 175 175 175 F F F 28 560-579 95 95 95 179 179 179 F F F 29 580-599 94 94 94 175 175 175 F F F Max gen: 97, Max flush: 183 Begin loop 49 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:35.986701504Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.015213404Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.015745938Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.015820205Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.015846146Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.015881536Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584} {"msg":"Crucible 40be0cd1-4bac-43ee-8580-0103a57fbddd has session id: 16550255-9d3e-4a7a-92d7-151af6a13251","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.017162803Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Upstairs opts: Upstairs UUID: 40be0cd1-4bac-43ee-8580-0103a57fbddd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.017225107Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Crucible stats registered with UUID: 40be0cd1-4bac-43ee-8580-0103a57fbddd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.017254845Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:36.017280085Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.01767153Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.017728022Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.017756739Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.017954481Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.01798448Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584} {"msg":"40be0cd1-4bac-43ee-8580-0103a57fbddd active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.018207662Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.018243934Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"client":"0","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.018271789Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"client":"1","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.018298792Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"client":"2","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.018478318Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.018548968Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.018577905Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:36.018977185Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:36.019022465Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:36.019058166Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.019240086Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.019280366Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"ds_connection connected from Ok(127.0.0.1:60111)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.019401305Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"ds_connection connected from Ok(127.0.0.1:65204)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.019434671Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.021294073Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"client":"1","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.021492856Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"client":"0","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.02195955Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:36.022329272Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.020520645Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"ds_connection connected from Ok(127.0.0.1:33317)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.020734118Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.023228416Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"client":"2","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"[0]R flush_numbers[0..12]: [179, 171, 181, 183, 183, 183, 180, 179, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.024488772Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"downstairs","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"[0]R generation[0..12]: [95, 92, 96, 97, 97, 97, 96, 95, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.024577467Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"downstairs","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.024611925Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"downstairs","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"[1]R flush_numbers[0..12]: [179, 171, 181, 183, 183, 183, 180, 179, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.024643448Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"downstairs","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"[1]R generation[0..12]: [95, 92, 96, 97, 97, 97, 96, 95, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.024673056Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"downstairs","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.02470502Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"downstairs","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"[2]R flush_numbers[0..12]: [179, 171, 181, 183, 183, 183, 180, 179, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.02473569Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"downstairs","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"[2]R generation[0..12]: [95, 92, 96, 97, 97, 97, 96, 95, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.02476562Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"downstairs","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.024797232Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"downstairs","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.024835378Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"downstairs","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.024862982Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"downstairs","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Next flush: 184","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.024888783Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"downstairs","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.024914484Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"":"downstairs","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.024941518Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.024965796Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"client":"0","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.025024162Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"client":"1","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.02505352Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"client":"2","":"downstairs","session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.025081074Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.025106885Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"40be0cd1-4bac-43ee-8580-0103a57fbddd is now active with session: 16550255-9d3e-4a7a-92d7-151af6a13251","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.025135873Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.025160181Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584,"session_id":"16550255-9d3e-4a7a-92d7-151af6a13251"} {"msg":"Wait for a query_work_queue command to finish before sending IO The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.025202505Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584} {"msg":"Activated sub_volume 40be0cd1-4bac-43ee-8580-0103a57fbddd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:46.025226512Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 539 len 4096 data: 8 02/30 Read block 166 len 32768 03/30 Read block 445 len 20480 04/30 Read block 10 len 36864 05/30 Read block 238 len 16384 06/30 Read block 194 len 28672 07/30 Flush 08/30 Write block 486 len 20480 data: 5 6 4 3 4 09/30 Read block 183 len 16384 10/30 Read block 232 len 40960 11/30 Read block 22 len 20480 12/30 Write block 454 len 20480 data: 9 8 10 13 14 13/30 Read block 72 len 28672 14/30 Read block 388 len 12288 15/30 Write block 367 len 8192 data: 10 9 16/30 Read block 103 len 16384 17/30 Read block 539 len 4096 18/30 Write block 466 len 8192 data: 9 8 19/30 Read block 509 len 16384 20/30 Read block 523 len 32768 21/30 Read block 143 len 20480 22/30 Read block 254 len 16384 23/30 Write block 79 len 4096 data: 4 24/30 Write block 43 len 36864 data: 5 4 7 6 7 5 6 6 8 25/30 Read block 456 len 4096 26/30 Flush 27/30 Write block 377 len 4096 data: 6 28/30 Read block 325 len 32768 29/30 Write block 269 len 40960 data: 9 9 9 9 9 9 8 9 7 8 30/30 Read block 81 len 12288 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Read 8 Done Done Sent false 1002 Acked Read 5 Done Done Sent false 1003 Acked Read 9 Done Done Sent false 1004 Acked Read 4 Done Done Sent false 1005 Acked Read 7 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Read 4 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Read 5 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Write 10 Sent Done Sent false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 30 32 Done 28 30 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:46.063920671Z","hostname":"bmat-EVT22200007-0000ead9","pid":1584} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 179 179 179 F F F 1 020-039 92 92 92 171 171 171 F F F 2 040-059 98 98 96 185 185 181 F F F <--- 3 060-079 98 98 97 185 185 183 F F F <--- 4 080-099 97 97 97 183 183 183 F F F 5 100-119 97 97 97 183 183 183 F F F 6 120-139 96 96 96 180 180 180 F F F 7 140-159 95 95 95 179 179 179 F F F 8 160-179 91 91 91 168 168 168 F F F 9 180-199 91 91 91 168 168 168 F F F 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 96 96 96 180 180 180 F F F 13 260-279 96 96 96 180 180 180 T T F <--- 14 280-299 94 94 94 178 178 178 F F F 15 300-319 90 90 90 167 167 167 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 95 95 95 179 179 179 F F F 18 360-379 98 98 95 185 185 179 T T F <--- 19 380-399 96 96 96 180 180 180 F F F 20 400-419 96 96 96 180 180 180 F F F 21 420-439 97 97 97 183 183 183 F F F 22 440-459 98 98 92 185 185 172 F F F <--- 23 460-479 98 98 92 185 185 169 F F F <--- 24 480-499 98 98 94 185 185 178 F F F <--- 25 500-519 94 94 94 178 178 178 F F F 26 520-539 98 98 93 184 184 173 F F F <--- 27 540-559 94 94 94 175 175 175 F F F 28 560-579 95 95 95 179 179 179 F F F 29 580-599 94 94 94 175 175 175 F F F Max gen: 98, Max flush: 185 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 179 179 179 F F F 1 020-039 92 92 92 171 171 171 F F F 2 040-059 98 98 98 185 185 185 F F F 3 060-079 98 98 98 185 185 185 F F F 4 080-099 97 97 97 183 183 183 F F F 5 100-119 97 97 97 183 183 183 F F F 6 120-139 96 96 96 180 180 180 F F F 7 140-159 95 95 95 179 179 179 F F F 8 160-179 91 91 91 168 168 168 F F F 9 180-199 91 91 91 168 168 168 F F F 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 96 96 96 180 180 180 F F F 13 260-279 99 99 99 186 186 186 F F F 14 280-299 94 94 94 178 178 178 F F F 15 300-319 90 90 90 167 167 167 F F F 16 320-339 88 88 88 163 163 163 F F F 17 340-359 95 95 95 179 179 179 F F F 18 360-379 99 99 99 186 186 186 F F F 19 380-399 96 96 96 180 180 180 F F F 20 400-419 96 96 96 180 180 180 F F F 21 420-439 97 97 97 183 183 183 F F F 22 440-459 98 98 98 185 185 185 F F F 23 460-479 98 98 98 185 185 185 F F F 24 480-499 98 98 98 185 185 185 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 98 98 98 184 184 184 F F F 27 540-559 94 94 94 175 175 175 F F F 28 560-579 95 95 95 179 179 179 F F F 29 580-599 94 94 94 175 175 175 F F F Max gen: 99, Max flush: 186 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.677330233Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.716870467Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.717623829Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.717777333Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.717809537Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.717852291Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592} {"msg":"Crucible 729fceea-e43c-4ebf-a2f3-20efff8acad0 has session id: 4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.718927831Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"Upstairs opts: Upstairs UUID: 729fceea-e43c-4ebf-a2f3-20efff8acad0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.719018691Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"Crucible stats registered with UUID: 729fceea-e43c-4ebf-a2f3-20efff8acad0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.71906385Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:50.719100823Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.719442049Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.719537658Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.719585042Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.719863203Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.719911458Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592} {"msg":"729fceea-e43c-4ebf-a2f3-20efff8acad0 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.72017001Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.720223426Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"client":"0","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.720278385Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"client":"1","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.720325308Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"client":"2","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.720370106Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.720414975Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.720461187Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:41310)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.720873884Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:58829)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.721012698Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:59542)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.721054441Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.722839105Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"client":"1","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.723088118Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"client":"0","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.723308233Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"client":"2","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.723762683Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.723812311Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"[0]R flush_numbers[0..12]: [179, 171, 185, 185, 183, 183, 180, 179, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.723927319Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"downstairs","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"[0]R generation[0..12]: [95, 92, 98, 98, 97, 97, 96, 95, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.723964994Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"downstairs","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.723997638Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"downstairs","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"[1]R flush_numbers[0..12]: [179, 171, 185, 185, 183, 183, 180, 179, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724037688Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"downstairs","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"[1]R generation[0..12]: [95, 92, 98, 98, 97, 97, 96, 95, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724067887Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"downstairs","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724099791Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"downstairs","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"[2]R flush_numbers[0..12]: [179, 171, 185, 185, 183, 183, 180, 179, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724130712Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"downstairs","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"[2]R generation[0..12]: [95, 92, 98, 98, 97, 97, 96, 95, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724160641Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"downstairs","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724192374Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"downstairs","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724218656Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"downstairs","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.72424607Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"downstairs","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.72427149Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"downstairs","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724297061Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"":"downstairs","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724324195Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724348703Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"client":"0","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724376047Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"client":"1","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.72440279Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"client":"2","":"downstairs","session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724433792Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724459382Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"729fceea-e43c-4ebf-a2f3-20efff8acad0 is now active with session: 4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.72448827Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:12:50.724512387Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592,"session_id":"4b0fe2e5-c08f-4e2d-bc0a-1e3ed83fd9ea"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724552577Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592} {"msg":"Activated sub_volume 729fceea-e43c-4ebf-a2f3-20efff8acad0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:50.724574621Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 44 len 4096 data: 5 02/30 Read block 581 len 20480 03/30 Read block 78 len 4096 04/30 Write block 43 len 16384 data: 6 6 8 7 05/30 Read block 181 len 12288 06/30 Read block 480 len 32768 07/30 Write block 586 len 20480 data: 8 4 4 4 5 08/30 Read block 206 len 36864 09/30 Read block 326 len 28672 10/30 Read block 47 len 24576 11/30 Flush 12/30 Write block 562 len 40960 data: 10 11 11 9 9 9 9 9 12 10 13/30 Write block 71 len 4096 data: 5 14/30 Write block 304 len 40960 data: 12 11 11 12 12 12 11 11 7 5 15/30 Read block 187 len 40960 16/30 Write block 581 len 20480 data: 9 9 9 11 9 17/30 Write block 344 len 40960 data: 5 5 7 6 6 6 5 5 3 4 18/30 Read block 2 len 24576 19/30 Read block 217 len 8192 20/30 Read block 520 len 16384 21/30 Read block 136 len 16384 22/30 Read block 582 len 16384 23/30 Read block 91 len 4096 24/30 Read block 431 len 12288 25/30 Read block 0 len 4096 26/30 Read block 493 len 40960 27/30 Read block 9 len 16384 28/30 Read block 158 len 4096 29/30 Write block 319 len 20480 data: 7 12 11 12 12 30/30 Read block 78 len 24576 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 1 Sent Done Done false 1003 Acked Write 4 Sent Done Done false 1004 Acked Read 3 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 1 29 Done 2 30 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:50.75924865Z","hostname":"bmat-EVT22200007-0000ead9","pid":1592} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 179 179 179 F F F 1 020-039 92 92 92 171 171 171 F F F 2 040-059 98 100 100 185 187 187 T F F <--- 3 060-079 98 98 98 185 185 185 F T T <--- 4 080-099 97 97 97 183 183 183 F F F 5 100-119 97 97 97 183 183 183 F F F 6 120-139 96 96 96 180 180 180 F F F 7 140-159 95 95 95 179 179 179 F F F 8 160-179 91 91 91 168 168 168 F F F 9 180-199 91 91 91 168 168 168 F F F 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 96 96 96 180 180 180 F F F 13 260-279 99 99 99 186 186 186 F F F 14 280-299 94 94 94 178 178 178 F F F 15 300-319 90 90 90 167 167 167 F T T <--- 16 320-339 88 88 88 163 163 163 F T T <--- 17 340-359 95 95 95 179 179 179 F T T <--- 18 360-379 99 99 99 186 186 186 F F F 19 380-399 96 96 96 180 180 180 F F F 20 400-419 96 96 96 180 180 180 F F F 21 420-439 97 97 97 183 183 183 F F F 22 440-459 98 98 98 185 185 185 F F F 23 460-479 98 98 98 185 185 185 F F F 24 480-499 98 98 98 185 185 185 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 98 98 98 184 184 184 F F F 27 540-559 94 94 94 175 175 175 F F F 28 560-579 95 95 95 179 179 179 F T T <--- 29 580-599 94 100 100 175 187 187 F T T <--- Max gen: 100, Max flush: 187 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 179 179 179 F F F 1 020-039 92 92 92 171 171 171 F F F 2 040-059 100 100 100 187 187 187 F F F 3 060-079 101 101 101 188 188 188 F F F 4 080-099 97 97 97 183 183 183 F F F 5 100-119 97 97 97 183 183 183 F F F 6 120-139 96 96 96 180 180 180 F F F 7 140-159 95 95 95 179 179 179 F F F 8 160-179 91 91 91 168 168 168 F F F 9 180-199 91 91 91 168 168 168 F F F 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 96 96 96 180 180 180 F F F 13 260-279 99 99 99 186 186 186 F F F 14 280-299 94 94 94 178 178 178 F F F 15 300-319 101 101 101 188 188 188 F F F 16 320-339 101 101 101 188 188 188 F F F 17 340-359 101 101 101 188 188 188 F F F 18 360-379 99 99 99 186 186 186 F F F 19 380-399 96 96 96 180 180 180 F F F 20 400-419 96 96 96 180 180 180 F F F 21 420-439 97 97 97 183 183 183 F F F 22 440-459 98 98 98 185 185 185 F F F 23 460-479 98 98 98 185 185 185 F F F 24 480-499 98 98 98 185 185 185 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 98 98 98 184 184 184 F F F 27 540-559 94 94 94 175 175 175 F F F 28 560-579 101 101 101 188 188 188 F F F 29 580-599 101 101 101 188 188 188 F F F Max gen: 101, Max flush: 188 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.364077712Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.398555614Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.399353624Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.399456658Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.399486687Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.399519642Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600} {"msg":"Crucible 988df9d1-f2f8-44aa-bfbd-6c6e843d5288 has session id: 8312599c-8a69-4315-ab7d-2001b2da5105","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.400849595Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"Upstairs opts: Upstairs UUID: 988df9d1-f2f8-44aa-bfbd-6c6e843d5288, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.400908823Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"Crucible stats registered with UUID: 988df9d1-f2f8-44aa-bfbd-6c6e843d5288","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.400935726Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:55.400964904Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.401494623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.40154895Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.401632976Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.401943942Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.401987599Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600} {"msg":"988df9d1-f2f8-44aa-bfbd-6c6e843d5288 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.402093408Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.402120492Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"client":"0","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.402145411Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"client":"1","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.40217008Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"client":"2","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.402194849Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.402441016Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.40251355Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"ds_connection connected from Ok(127.0.0.1:35625)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.402544641Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"ds_connection connected from Ok(127.0.0.1:45676)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.40257441Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"ds_connection connected from Ok(127.0.0.1:63367)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.402603488Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.404182163Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"client":"2","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.404379504Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"client":"0","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.405256921Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.405379514Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"client":"1","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.405579991Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"[0]R flush_numbers[0..12]: [179, 171, 187, 188, 183, 183, 180, 179, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406169548Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"downstairs","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"[0]R generation[0..12]: [95, 92, 100, 101, 97, 97, 96, 95, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406204287Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"downstairs","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.40623626Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"downstairs","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"[1]R flush_numbers[0..12]: [179, 171, 187, 188, 183, 183, 180, 179, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406266771Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"downstairs","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"[1]R generation[0..12]: [95, 92, 100, 101, 97, 97, 96, 95, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406296409Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"downstairs","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406327711Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"downstairs","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"[2]R flush_numbers[0..12]: [179, 171, 187, 188, 183, 183, 180, 179, 168, 168, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406357991Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"downstairs","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"[2]R generation[0..12]: [95, 92, 100, 101, 97, 97, 96, 95, 91, 91, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406387359Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"downstairs","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406418892Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"downstairs","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406444613Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"downstairs","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406477348Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"downstairs","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406502988Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"downstairs","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406528509Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"":"downstairs","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406555502Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.40657967Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"client":"0","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406606253Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"client":"1","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406632725Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"client":"2","":"downstairs","session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406659969Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406685309Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"988df9d1-f2f8-44aa-bfbd-6c6e843d5288 is now active with session: 8312599c-8a69-4315-ab7d-2001b2da5105","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406714106Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"session_id":"Wait for a query_work_queue command to finish before sending IO 8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406755609Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600,"session_id":"8312599c-8a69-4315-ab7d-2001b2da5105"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406781309Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600} {"msg":"Activated sub_volume 988df9d1-f2f8-44aa-bfbd-6c6e843d5288","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:12:55.406803283Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 153 len 36864 data: 7 7 5 6 7 6 7 8 11 02/30 Write block 30 len 12288 data: 7 7 7 03/30 Read block 186 len 28672 04/30 Read block 483 len 40960 05/30 Read block 533 len 24576 06/30 Write block 302 len 4096 data: 8 07/30 Write block 288 len 28672 data: 10 8 9 12 11 6 8 08/30 Read block 439 len 36864 09/30 Write block 522 len 8192 data: 7 6 10/30 Read block 590 len 36864 11/30 Read block 166 len 8192 12/30 Write block 169 len 32768 data: 9 8 9 8 8 7 7 9 13/30 Read block 149 len 20480 14/30 Write block 379 len 28672 data: 8 7 8 8 6 6 9 15/30 Read block 598 len 4096 16/30 Read block 446 len 36864 17/30 Read block 472 len 28672 18/30 Read block 451 len 36864 19/30 Write block 482 len 28672 data: 8 8 6 5 6 7 5 20/30 Write block 372 len 4096 data: 8 21/30 Write block 528 len 24576 data: 8 10 9 9 9 11 22/30 Read block 485 len 4096 23/30 Flush 24/30 Write block 31 len 4096 data: 8 25/30 Write block 65 len 36864 data: 7 5 4 7 8 9 6 6 6 26/30 Write block 190 len 20480 data: 10 12 11 14 13 27/30 Write block 396 len 8192 data: 9 10 28/30 Write block 94 len 36864 data: 10 10 12 13 12 12 11 6 7 29/30 Write block 384 len 40960 data: 7 10 7 6 6 5 6 8 9 7 30/30 Write block 303 len 8192 data: 9 13 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 19 3 25 Done 27 11 27 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:12:55.437061089Z","hostname":"bmat-EVT22200007-0000ead9","pid":1600} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 179 179 179 F F F 1 020-039 102 92 102 189 171 189 T T T <--- 2 040-059 100 100 100 187 187 187 F F F 3 060-079 101 101 101 188 188 188 T F T <--- 4 080-099 97 97 97 183 183 183 T F T <--- 5 100-119 97 97 97 183 183 183 T F T <--- 6 120-139 96 96 96 180 180 180 F F F 7 140-159 102 95 102 189 179 189 F T F <--- 8 160-179 102 91 102 189 168 189 F T F <--- 9 180-199 91 91 91 168 168 168 T F T <--- 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 96 96 96 180 180 180 F F F 13 260-279 99 99 99 186 186 186 F F F 14 280-299 102 94 102 189 178 189 F T F <--- 15 300-319 102 101 102 189 188 189 T T T <--- 16 320-339 101 101 101 188 188 188 F F F 17 340-359 101 101 101 188 188 188 F F F 18 360-379 102 99 102 189 186 189 F F F <--- 19 380-399 102 96 102 189 180 189 T F T <--- 20 400-419 96 96 96 180 180 180 F F F 21 420-439 97 97 97 183 183 183 F F F 22 440-459 98 98 98 185 185 185 F F F 23 460-479 98 98 98 185 185 185 F F F 24 480-499 102 98 102 189 185 189 F F F <--- 25 500-519 94 94 94 178 178 178 F F F 26 520-539 102 98 102 189 184 189 F T F <--- 27 540-559 94 94 94 175 175 175 F F F 28 560-579 101 101 101 188 188 188 F F F 29 580-599 101 101 101 188 188 188 F F F Max gen: 102, Max flush: 189 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 95 95 95 179 179 179 F F F 1 020-039 103 103 103 190 190 190 F F F 2 040-059 100 100 100 187 187 187 F F F 3 060-079 103 103 103 190 190 190 F F F 4 080-099 103 103 103 190 190 190 F F F 5 100-119 103 103 103 190 190 190 F F F 6 120-139 96 96 96 180 180 180 F F F 7 140-159 102 102 102 189 189 189 F F F 8 160-179 102 102 102 189 189 189 F F F 9 180-199 103 103 103 190 190 190 F F F 10 200-219 89 89 89 164 164 164 F F F 11 220-239 88 88 88 163 163 163 F F F 12 240-259 96 96 96 180 180 180 F F F 13 260-279 99 99 99 186 186 186 F F F 14 280-299 102 102 102 189 189 189 F F F 15 300-319 103 103 103 190 190 190 F F F 16 320-339 101 101 101 188 188 188 F F F 17 340-359 101 101 101 188 188 188 F F F 18 360-379 102 102 102 189 189 189 F F F 19 380-399 103 103 103 190 190 190 F F F 20 400-419 96 96 96 180 180 180 F F F 21 420-439 97 97 97 183 183 183 F F F 22 440-459 98 98 98 185 185 185 F F F 23 460-479 98 98 98 185 185 185 F F F 24 480-499 102 102 102 189 189 189 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 102 102 102 189 189 189 F F F 27 540-559 94 94 94 175 175 175 F F F 28 560-579 101 101 101 188 188 188 F F F 29 580-599 101 101 101 188 188 188 F F F Max gen: 103, Max flush: 190 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.093304516Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.120743932Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.121252359Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.121369871Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.121410772Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.121466102Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608} {"msg":"Crucible 953bde6f-4e4e-4def-a4c3-64c2cccdaa87 has session id: 19d5c2ab-040f-4f06-8bcd-6427a7967791","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.122616219Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"Upstairs opts: Upstairs UUID: 953bde6f-4e4e-4def-a4c3-64c2cccdaa87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.122703752Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"Crucible stats registered with UUID: 953bde6f-4e4e-4def-a4c3-64c2cccdaa87","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.122745685Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:00.122780775Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.123263821Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.123401013Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.123459889Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.123646539Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.123704383Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608} {"msg":"953bde6f-4e4e-4def-a4c3-64c2cccdaa87 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.123959088Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.124018545Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"client":"0","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.124062232Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"client":"1","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.124110978Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"client":"2","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.124151819Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.124191237Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"ds_connection connected from Ok(127.0.0.1:58109)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.124342717Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"ds_connection connected from Ok(127.0.0.1:39336)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.124402736Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.124662611Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:00.125011171Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:00.12507665Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:00.12513695Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.125187059Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.125246346Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.126451442Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"client":"2","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.126747839Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"client":"0","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.127300052Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:00.127389169Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.127020655Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"ds_connection connected from Ok(127.0.0.1:54345)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.127419665Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.129404365Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"client":"1","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"[0]R flush_numbers[0..12]: [179, 190, 187, 190, 190, 190, 180, 189, 189, 190, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130428031Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"downstairs","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"[0]R generation[0..12]: [95, 103, 100, 103, 103, 103, 96, 102, 102, 103, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.13048302Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"downstairs","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130519583Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"downstairs","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"[1]R flush_numbers[0..12]: [179, 190, 187, 190, 190, 190, 180, 189, 189, 190, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130554081Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"downstairs","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"[1]R generation[0..12]: [95, 103, 100, 103, 103, 103, 96, 102, 102, 103, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130587627Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"downstairs","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130622897Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"downstairs","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"[2]R flush_numbers[0..12]: [179, 190, 187, 190, 190, 190, 180, 189, 189, 190, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130656474Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"downstairs","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"[2]R generation[0..12]: [95, 103, 100, 103, 103, 103, 96, 102, 102, 103, 89, 88]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.13068974Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"downstairs","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130724719Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"downstairs","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130753506Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"downstairs","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130791261Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"downstairs","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130819877Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"downstairs","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130848223Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"":"downstairs","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130877952Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130904685Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"client":"0","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130934334Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"client":"1","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.130963692Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"client":"2","":"downstairs","session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.13101829Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.131047708Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"953bde6f-4e4e-4def-a4c3-64c2cccdaa87 is now active with session: 19d5c2ab-040f-4f06-8bcd-6427a7967791","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.131079601Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.131106234Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608,"session_id":"19d5c2ab-040f-4f06-8bcd-6427a7967791"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2026-03-04T01:13:10.13113426Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608} {"msg":"Activated sub_volume 953bde6f-4e4e-4def-a4c3-64c2cccdaa87","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:10.131173698Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 19 len 28672 data: 6 7 8 9 7 7 5 02/30 Read block 473 len 28672 03/30 Write block 337 len 36864 data: 8 6 6 7 9 8 5 6 6 04/30 Read block 39 len 28672 05/30 Write block 399 len 20480 data: 9 12 14 12 10 06/30 Flush 07/30 Flush 08/30 Write block 174 len 12288 data: 8 8 10 09/30 Write block 208 len 8192 data: 11 11 10/30 Read block 60 len 40960 11/30 Write block 547 len 24576 data: 7 7 6 8 11 12 12/30 Read block 495 len 40960 13/30 Write block 429 len 12288 data: 10 10 10 14/30 Write block 120 len 40960 data: 12 10 8 6 6 6 7 9 8 10 15/30 Read block 252 len 40960 16/30 Write block 110 len 40960 data: 7 7 7 13 12 10 11 13 12 11 17/30 Write block 198 len 24576 data: 10 8 10 10 9 9 18/30 Flush 19/30 Write block 426 len 24576 data: 9 8 8 11 11 11 20/30 Flush 21/30 Write block 5 len 32768 data: 11 13 12 8 9 7 8 6 22/30 Read block 254 len 12288 23/30 Write block 322 len 8192 data: 13 13 24/30 Flush 25/30 Write block 27 len 32768 data: 5 6 7 8 9 8 7 7 26/30 Flush 27/30 Write block 173 len 32768 data: 9 9 9 11 11 7 8 12 28/30 Write block 220 len 24576 data: 6 4 5 5 6 6 29/30 Read block 422 len 40960 30/30 Read block 321 len 8192 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 10 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 24 0 24 Done 24 0 24 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1005)) Some(JobId(1025)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:10.171306565Z","hostname":"bmat-EVT22200007-0000ead9","pid":1608} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 195 191 195 F F F <--- 1 020-039 104 104 104 196 191 196 F F F <--- 2 040-059 100 100 100 187 187 187 F F F 3 060-079 103 103 103 190 190 190 F F F 4 080-099 103 103 103 190 190 190 F F F 5 100-119 104 103 104 193 190 193 F F F <--- 6 120-139 104 96 104 193 180 193 F F F <--- 7 140-159 102 102 102 189 189 189 F F F 8 160-179 104 102 104 193 189 193 T F T <--- 9 180-199 104 103 104 193 190 193 T F T <--- 10 200-219 104 89 104 193 164 193 F F F <--- 11 220-239 88 88 88 163 163 163 T F T <--- 12 240-259 96 96 96 180 180 180 F F F 13 260-279 99 99 99 186 186 186 F F F 14 280-299 102 102 102 189 189 189 F F F 15 300-319 103 103 103 190 190 190 F F F 16 320-339 104 104 104 195 191 195 F F F <--- 17 340-359 104 104 104 191 191 191 F F F 18 360-379 102 102 102 189 189 189 F F F 19 380-399 104 104 104 191 191 191 F F F 20 400-419 104 104 104 191 191 191 F F F 21 420-439 104 97 104 194 183 194 F F F <--- 22 440-459 98 98 98 185 185 185 F F F 23 460-479 98 98 98 185 185 185 F F F 24 480-499 102 102 102 189 189 189 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 102 102 102 189 189 189 F F F 27 540-559 104 94 104 193 175 193 F F F <--- 28 560-579 101 101 101 188 188 188 F F F 29 580-599 101 101 101 188 188 188 F F F Max gen: 104, Max flush: 196 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 195 195 195 F F F 1 020-039 104 104 104 196 196 196 F F F 2 040-059 100 100 100 187 187 187 F F F 3 060-079 103 103 103 190 190 190 F F F 4 080-099 103 103 103 190 190 190 F F F 5 100-119 104 104 104 193 193 193 F F F 6 120-139 104 104 104 193 193 193 F F F 7 140-159 102 102 102 189 189 189 F F F 8 160-179 105 105 105 197 197 197 F F F 9 180-199 105 105 105 197 197 197 F F F 10 200-219 104 104 104 193 193 193 F F F 11 220-239 105 105 105 197 197 197 F F F 12 240-259 96 96 96 180 180 180 F F F 13 260-279 99 99 99 186 186 186 F F F 14 280-299 102 102 102 189 189 189 F F F 15 300-319 103 103 103 190 190 190 F F F 16 320-339 104 104 104 195 195 195 F F F 17 340-359 104 104 104 191 191 191 F F F 18 360-379 102 102 102 189 189 189 F F F 19 380-399 104 104 104 191 191 191 F F F 20 400-419 104 104 104 191 191 191 F F F 21 420-439 104 104 104 194 194 194 F F F 22 440-459 98 98 98 185 185 185 F F F 23 460-479 98 98 98 185 185 185 F F F 24 480-499 102 102 102 189 189 189 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 102 102 102 189 189 189 F F F 27 540-559 104 104 104 193 193 193 F F F 28 560-579 101 101 101 188 188 188 F F F 29 580-599 101 101 101 188 188 188 F F F Max gen: 105, Max flush: 197 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.76742862Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.796453175Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.79704158Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.797155796Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.797206095Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.797269912Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616} {"msg":"Crucible af20d766-30a9-4025-ab5d-0445a6bd769d has session id: 6680194c-c0c9-41ed-be61-5d546e23ea8e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.798366793Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"Upstairs opts: Upstairs UUID: af20d766-30a9-4025-ab5d-0445a6bd769d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.798456661Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"Crucible stats registered with UUID: af20d766-30a9-4025-ab5d-0445a6bd769d","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.79851174Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:14.798555567Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.798928565Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.79905193Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.799104664Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.799269621Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.799320211Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616} {"msg":"af20d766-30a9-4025-ab5d-0445a6bd769d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.79936011Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.799401452Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"client":"0","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.799443525Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"client":"1","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.799487913Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"client":"2","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.799533333Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.799586478Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.799631658Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45482)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.799675264Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36694)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.799726545Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:62609)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.799810201Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.801822796Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"client":"0","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.802271674Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"client":"2","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.802431962Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"client":"1","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.802817555Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803230182Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"[0]R flush_numbers[0..12]: [195, 196, 187, 190, 190, 193, 193, 189, 197, 197, 193, 197]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803426801Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"downstairs","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"[0]R generation[0..12]: [104, 104, 100, 103, 103, 104, 104, 102, 105, 105, 104, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803464897Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"downstairs","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803502391Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"downstairs","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"[1]R flush_numbers[0..12]: [195, 196, 187, 190, 190, 193, 193, 189, 197, 197, 193, 197]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803545316Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"downstairs","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"[1]R generation[0..12]: [104, 104, 100, 103, 103, 104, 104, 102, 105, 105, 104, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803591798Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"downstairs","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803628611Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"downstairs","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"[2]R flush_numbers[0..12]: [195, 196, 187, 190, 190, 193, 193, 189, 197, 197, 193, 197]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803667267Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"downstairs","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"[2]R generation[0..12]: [104, 104, 100, 103, 103, 104, 104, 102, 105, 105, 104, 105]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.80369881Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"downstairs","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803728178Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"downstairs","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803752456Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"downstairs","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803777055Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"downstairs","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803800521Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"downstairs","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803824188Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"":"downstairs","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803848967Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803871462Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"client":"0","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803896131Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"client":"1","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803920569Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"client":"2","":"downstairs","session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803945789Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.803969245Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"af20d766-30a9-4025-ab5d-0445a6bd769d is now active with session: 6680194c-c0c9-41ed-be61-5d546e23ea8e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.804000207Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.804022821Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616,"session_id":"6680194c-c0c9-41ed-be61-5d546e23ea8e"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.804046639Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616} {"msg":"Activated sub_volume af20d766-30a9-4025-ab5d-0445a6bd769d","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:14.804066909Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 227 len 28672 data: 6 7 8 9 9 9 10 02/30 Write block 132 len 36864 data: 9 10 12 11 12 11 12 14 12 03/30 Read block 327 len 32768 04/30 Write block 533 len 40960 data: 12 11 10 12 10 9 9 7 8 9 05/30 Flush 06/30 Write block 542 len 24576 data: 10 9 6 7 6 8 07/30 Read block 53 len 16384 08/30 Read block 312 len 4096 09/30 Write block 234 len 12288 data: 11 8 9 10/30 Write block 206 len 20480 data: 11 12 12 12 10 11/30 Flush 12/30 Write block 45 len 40960 data: 9 8 8 6 7 7 9 9 9 8 13/30 Read block 54 len 36864 14/30 Write block 338 len 24576 data: 7 7 8 10 9 6 15/30 Flush 16/30 Write block 236 len 40960 data: 10 9 9 11 11 11 9 8 7 8 17/30 Read block 290 len 24576 18/30 Read block 219 len 8192 19/30 Read block 548 len 28672 20/30 Read block 167 len 24576 21/30 Read block 496 len 8192 22/30 Read block 389 len 12288 23/30 Write block 21 len 40960 data: 9 10 8 8 6 4 6 7 8 9 24/30 Write block 414 len 8192 data: 6 7 25/30 Read block 406 len 20480 26/30 Write block 253 len 8192 data: 6 7 27/30 Read block 592 len 24576 28/30 Read block 525 len 8192 29/30 Read block 528 len 20480 30/30 Read block 110 len 20480 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Sent false 1001 Acked Write 9 Done Done Sent false 1002 Acked Read 8 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 6 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 9 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1014)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:14.840190388Z","hostname":"bmat-EVT22200007-0000ead9","pid":1616} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 195 195 195 F F F 1 020-039 104 104 104 196 196 196 T T F <--- 2 040-059 106 106 100 200 200 187 F F F <--- 3 060-079 103 103 103 190 190 190 F F F 4 080-099 103 103 103 190 190 190 F F F 5 100-119 104 104 104 193 193 193 F F F 6 120-139 106 106 104 198 198 193 F F F <--- 7 140-159 106 106 102 198 198 189 F F F <--- 8 160-179 105 105 105 197 197 197 F F F 9 180-199 105 105 105 197 197 197 F F F 10 200-219 106 106 104 199 199 193 F F F <--- 11 220-239 106 106 105 199 199 197 T T F <--- 12 240-259 96 96 96 180 180 180 T T F <--- 13 260-279 99 99 99 186 186 186 F F F 14 280-299 102 102 102 189 189 189 F F F 15 300-319 103 103 103 190 190 190 F F F 16 320-339 106 106 104 200 200 195 F F F <--- 17 340-359 106 106 104 200 200 191 F F F <--- 18 360-379 102 102 102 189 189 189 F F F 19 380-399 104 104 104 191 191 191 F F F 20 400-419 104 104 104 191 191 191 T T F <--- 21 420-439 104 104 104 194 194 194 F F F 22 440-459 98 98 98 185 185 185 F F F 23 460-479 98 98 98 185 185 185 F F F 24 480-499 102 102 102 189 189 189 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 106 106 102 198 198 189 F F F <--- 27 540-559 106 106 104 199 199 193 F F F <--- 28 560-579 101 101 101 188 188 188 F F F 29 580-599 101 101 101 188 188 188 F F F Max gen: 106, Max flush: 200 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 195 195 195 F F F 1 020-039 107 107 107 201 201 201 F F F 2 040-059 106 106 106 200 200 200 F F F 3 060-079 103 103 103 190 190 190 F F F 4 080-099 103 103 103 190 190 190 F F F 5 100-119 104 104 104 193 193 193 F F F 6 120-139 106 106 106 198 198 198 F F F 7 140-159 106 106 106 198 198 198 F F F 8 160-179 105 105 105 197 197 197 F F F 9 180-199 105 105 105 197 197 197 F F F 10 200-219 106 106 106 199 199 199 F F F 11 220-239 107 107 107 201 201 201 F F F 12 240-259 107 107 107 201 201 201 F F F 13 260-279 99 99 99 186 186 186 F F F 14 280-299 102 102 102 189 189 189 F F F 15 300-319 103 103 103 190 190 190 F F F 16 320-339 106 106 106 200 200 200 F F F 17 340-359 106 106 106 200 200 200 F F F 18 360-379 102 102 102 189 189 189 F F F 19 380-399 104 104 104 191 191 191 F F F 20 400-419 107 107 107 201 201 201 F F F 21 420-439 104 104 104 194 194 194 F F F 22 440-459 98 98 98 185 185 185 F F F 23 460-479 98 98 98 185 185 185 F F F 24 480-499 102 102 102 189 189 189 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 106 106 106 198 198 198 F F F 27 540-559 106 106 106 199 199 199 F F F 28 560-579 101 101 101 188 188 188 F F F 29 580-599 101 101 101 188 188 188 F F F Max gen: 107, Max flush: 201 Begin loop 54 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.465724615Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.49741648Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.497947892Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.498039604Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.498071547Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.498129823Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624} {"msg":"Crucible d1e5302c-0fe9-451b-8b60-b27117ee2ff1 has session id: 935c4db8-8ba7-4430-9601-85f894e71a03","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.499241704Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"Upstairs opts: Upstairs UUID: d1e5302c-0fe9-451b-8b60-b27117ee2ff1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.499317584Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"Crucible stats registered with UUID: d1e5302c-0fe9-451b-8b60-b27117ee2ff1","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.499357383Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:19.499392502Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.49987084Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.499941169Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.49998707Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.500130874Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.500170854Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624} {"msg":"d1e5302c-0fe9-451b-8b60-b27117ee2ff1 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.500311051Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.500343826Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"client":"0","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.50037122Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"client":"1","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.500398695Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"client":"2","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.500425428Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.500468743Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.50050249Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"ds_connection connected from Ok(127.0.0.1:61079)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.500661685Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"ds_connection connected from Ok(127.0.0.1:40378)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.500702035Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"ds_connection connected from Ok(127.0.0.1:37806)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.500842603Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.502506718Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"client":"1","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.502837042Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"client":"2","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.503282775Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.503521648Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.503571166Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"client":"0","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"[0]R flush_numbers[0..12]: [195, 201, 200, 190, 190, 193, 198, 198, 197, 197, 199, 201]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.504640823Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"downstairs","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"[0]R generation[0..12]: [104, 107, 106, 103, 103, 104, 106, 106, 105, 105, 106, 107]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.504686193Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"downstairs","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.504725261Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"downstairs","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"[1]R flush_numbers[0..12]: [195, 201, 200, 190, 190, 193, 198, 198, 197, 197, 199, 201]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.504762835Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"downstairs","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"[1]R generation[0..12]: [104, 107, 106, 103, 103, 104, 106, 106, 105, 105, 106, 107]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.504800229Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"downstairs","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.504839206Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"downstairs","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"[2]R flush_numbers[0..12]: [195, 201, 200, 190, 190, 193, 198, 198, 197, 197, 199, 201]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.50487661Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"downstairs","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"[2]R generation[0..12]: [104, 107, 106, 103, 103, 104, 106, 106, 105, 105, 106, 107]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.504913704Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"downstairs","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.50495224Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"downstairs","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.504983562Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"downstairs","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.505041877Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"downstairs","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.505081115Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"downstairs","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.505112798Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.505162416Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.505192395Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"client":"0","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.505225251Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"client":"1","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.505257565Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"client":"2","":"downstairs","session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.50529087Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.505321742Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"d1e5302c-0fe9-451b-8b60-b27117ee2ff1 is now active with session: 935c4db8-8ba7-4430-9601-85f894e71a03","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.505357182Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.50538645Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624,"session_id":"935c4db8-8ba7-4430-9601-85f894e71a03"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.505417491Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624} {"msg":"Activated sub_volume d1e5302c-0fe9-451b-8b60-b27117ee2ff1","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:19.505444094Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 421 len 4096 data: 4 02/30 Write block 89 len 4096 data: 9 03/30 Flush 04/30 Write block 436 len 8192 data: 12 15 05/30 Read block 15 len 40960 06/30 Read block 318 len 40960 07/30 Write block 164 len 36864 data: 12 13 10 11 13 10 9 10 9 08/30 Write block 461 len 28672 data: 9 11 10 9 8 10 9 09/30 Read block 69 len 32768 10/30 Read block 22 len 16384 11/30 Read block 50 len 32768 12/30 Read block 224 len 16384 13/30 Read block 148 len 12288 14/30 Read block 41 len 40960 15/30 Read block 445 len 24576 16/30 Read block 548 len 12288 17/30 Read block 468 len 24576 18/30 Write block 73 len 12288 data: 7 6 6 19/30 Read block 13 len 24576 20/30 Read block 545 len 32768 21/30 Write block 373 len 8192 data: 8 8 22/30 Read block 398 len 24576 23/30 Read block 102 len 4096 24/30 Write block 268 len 20480 data: 9 10 10 10 10 25/30 Write block 174 len 24576 data: 10 10 12 12 8 9 26/30 Read block 410 len 28672 27/30 Read block 384 len 16384 28/30 Write block 367 len 20480 data: 11 10 8 9 8 29/30 Read block 499 len 28672 30/30 Write block 489 len 40960 data: 4 5 4 5 5 7 9 8 8 7 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:19.538635087Z","hostname":"bmat-EVT22200007-0000ead9","pid":1624} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 195 195 195 F F F 1 020-039 107 107 107 201 201 201 F F F 2 040-059 106 106 106 200 200 200 F F F 3 060-079 103 103 103 190 190 190 F T T <--- 4 080-099 103 108 108 190 202 202 T F F <--- 5 100-119 104 104 104 193 193 193 F F F 6 120-139 106 106 106 198 198 198 F F F 7 140-159 106 106 106 198 198 198 F F F 8 160-179 105 105 105 197 197 197 F T T <--- 9 180-199 105 105 105 197 197 197 F F F 10 200-219 106 106 106 199 199 199 F F F 11 220-239 107 107 107 201 201 201 F F F 12 240-259 107 107 107 201 201 201 F F F 13 260-279 99 99 99 186 186 186 F T T <--- 14 280-299 102 102 102 189 189 189 F F F 15 300-319 103 103 103 190 190 190 F F F 16 320-339 106 106 106 200 200 200 F F F 17 340-359 106 106 106 200 200 200 F F F 18 360-379 102 102 102 189 189 189 F T T <--- 19 380-399 104 104 104 191 191 191 F F F 20 400-419 107 107 107 201 201 201 F F F 21 420-439 104 108 108 194 202 202 T T T <--- 22 440-459 98 98 98 185 185 185 F F F 23 460-479 98 98 98 185 185 185 F T T <--- 24 480-499 102 102 102 189 189 189 F T T <--- 25 500-519 94 94 94 178 178 178 F F F 26 520-539 106 106 106 198 198 198 F F F 27 540-559 106 106 106 199 199 199 F F F 28 560-579 101 101 101 188 188 188 F F F 29 580-599 101 101 101 188 188 188 F F F Max gen: 108, Max flush: 202 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 195 195 195 F F F 1 020-039 107 107 107 201 201 201 F F F 2 040-059 106 106 106 200 200 200 F F F 3 060-079 109 109 109 203 203 203 F F F 4 080-099 108 108 108 202 202 202 F F F 5 100-119 104 104 104 193 193 193 F F F 6 120-139 106 106 106 198 198 198 F F F 7 140-159 106 106 106 198 198 198 F F F 8 160-179 109 109 109 203 203 203 F F F 9 180-199 105 105 105 197 197 197 F F F 10 200-219 106 106 106 199 199 199 F F F 11 220-239 107 107 107 201 201 201 F F F 12 240-259 107 107 107 201 201 201 F F F 13 260-279 109 109 109 203 203 203 F F F 14 280-299 102 102 102 189 189 189 F F F 15 300-319 103 103 103 190 190 190 F F F 16 320-339 106 106 106 200 200 200 F F F 17 340-359 106 106 106 200 200 200 F F F 18 360-379 109 109 109 203 203 203 F F F 19 380-399 104 104 104 191 191 191 F F F 20 400-419 107 107 107 201 201 201 F F F 21 420-439 109 109 109 203 203 203 F F F 22 440-459 98 98 98 185 185 185 F F F 23 460-479 109 109 109 203 203 203 F F F 24 480-499 109 109 109 203 203 203 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 106 106 106 198 198 198 F F F 27 540-559 106 106 106 199 199 199 F F F 28 560-579 101 101 101 188 188 188 F F F 29 580-599 101 101 101 188 188 188 F F F Max gen: 109, Max flush: 203 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.189192044Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.222304241Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.223198201Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.223291756Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.223330744Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.223380843Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632} {"msg":"Crucible 49cd9b50-a0ec-4a1e-a9d8-536123fa169a has session id: e4f35b9a-27d9-476a-b233-3c2dec72b7b0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.224578454Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"Upstairs opts: Upstairs UUID: 49cd9b50-a0ec-4a1e-a9d8-536123fa169a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.224648843Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"Crucible stats registered with UUID: 49cd9b50-a0ec-4a1e-a9d8-536123fa169a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.224688211Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:24.224723201Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.225124565Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.225193652Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.225239252Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.225416944Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.225457344Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632} {"msg":"49cd9b50-a0ec-4a1e-a9d8-536123fa169a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.225559717Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.225602181Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"client":"0","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.225646058Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"client":"1","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.225689844Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"client":"2","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.225732659Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.225783229Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.225828258Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:54971)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.225893017Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:63264)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.226172089Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:42875)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.226246356Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.227639705Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"client":"0","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.227704082Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"client":"2","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.228348598Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.228734783Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.229011411Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"client":"1","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"[0]R flush_numbers[0..12]: [195, 201, 200, 203, 202, 193, 198, 198, 203, 197, 199, 201]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.229892705Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"downstairs","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"[0]R generation[0..12]: [104, 107, 106, 109, 108, 104, 106, 106, 109, 105, 106, 107]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.229939358Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"downstairs","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.229982523Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"downstairs","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"[1]R flush_numbers[0..12]: [195, 201, 200, 203, 202, 193, 198, 198, 203, 197, 199, 201]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230013976Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"downstairs","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"[1]R generation[0..12]: [104, 107, 106, 109, 108, 104, 106, 106, 109, 105, 106, 107]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230051029Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"downstairs","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230083243Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"downstairs","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"[2]R flush_numbers[0..12]: [195, 201, 200, 203, 202, 193, 198, 198, 203, 197, 199, 201]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230123713Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"downstairs","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"[2]R generation[0..12]: [104, 107, 106, 109, 108, 104, 106, 106, 109, 105, 106, 107]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230162129Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"downstairs","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230194553Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"downstairs","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230220865Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"downstairs","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230248239Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"downstairs","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.23027403Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"downstairs","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230299922Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"":"downstairs","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230327075Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230351634Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"client":"0","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230378577Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"client":"1","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"Transition from WaitQuorum to Active"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230419148Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"client":"2","":"downstairs","session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230447794Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230473706Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"49cd9b50-a0ec-4a1e-a9d8-536123fa169a is now active with session: e4f35b9a-27d9-476a-b233-3c2dec72b7b0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230507022Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.23053155Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632,"session_id":"e4f35b9a-27d9-476a-b233-3c2dec72b7b0"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230557361Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632} {"msg":"Activated sub_volume 49cd9b50-a0ec-4a1e-a9d8-536123fa169a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:24.230579565Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 575 len 36864 data: 9 7 8 9 8 9 10 10 10 02/30 Write block 305 len 24576 data: 12 12 13 13 13 12 03/30 Write block 326 len 4096 data: 7 04/30 Read block 27 len 32768 05/30 Write block 12 len 16384 data: 7 7 6 6 06/30 Write block 38 len 32768 data: 6 5 4 4 4 7 7 10 07/30 Read block 202 len 8192 08/30 Read block 457 len 32768 09/30 Read block 193 len 28672 10/30 Read block 191 len 36864 11/30 Write block 579 len 32768 data: 9 10 11 11 11 12 10 9 12/30 Write block 434 len 16384 data: 9 11 13 16 13/30 Read block 493 len 12288 14/30 Read block 15 len 4096 15/30 Write block 27 len 36864 data: 7 8 9 10 10 9 8 8 7 16/30 Write block 97 len 8192 data: 14 13 17/30 Write block 347 len 4096 data: 7 18/30 Read block 78 len 16384 19/30 Write block 54 len 16384 data: 9 8 10 9 20/30 Write block 19 len 24576 data: 7 8 10 11 9 9 21/30 Flush 22/30 Write block 191 len 24576 data: 13 12 15 14 10 11 23/30 Write block 403 len 40960 data: 11 13 12 12 11 11 9 7 7 7 24/30 Write block 280 len 20480 data: 7 7 6 7 10 25/30 Read block 248 len 40960 26/30 Read block 197 len 4096 27/30 Write block 310 len 8192 data: 13 12 28/30 Write block 49 len 32768 data: 8 8 10 10 10 10 9 11 29/30 Write block 470 len 16384 data: 7 8 8 9 30/30 Read block 226 len 36864 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Sent Done false 1001 Acked Write 6 Done Sent Done false 1002 Acked Write 1 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Write 8 Done Sent Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:24.262529191Z","hostname":"bmat-EVT22200007-0000ead9","pid":1632} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 104 110 204 195 204 F F F <--- 1 020-039 110 107 110 204 201 204 F F F <--- 2 040-059 110 106 110 204 200 204 T F T <--- 3 060-079 109 109 109 203 203 203 F F F 4 080-099 110 108 110 204 202 204 F F F <--- 5 100-119 104 104 104 193 193 193 F F F 6 120-139 106 106 106 198 198 198 F F F 7 140-159 106 106 106 198 198 198 F F F 8 160-179 109 109 109 203 203 203 F F F 9 180-199 105 105 105 197 197 197 T F T <--- 10 200-219 106 106 106 199 199 199 F F F 11 220-239 107 107 107 201 201 201 F F F 12 240-259 107 107 107 201 201 201 F F F 13 260-279 109 109 109 203 203 203 F F F 14 280-299 102 102 102 189 189 189 T F T <--- 15 300-319 110 103 110 204 190 204 T F T <--- 16 320-339 110 106 110 204 200 204 F F F <--- 17 340-359 110 106 110 204 200 204 F F F <--- 18 360-379 109 109 109 203 203 203 F F F 19 380-399 104 104 104 191 191 191 F F F 20 400-419 107 107 107 201 201 201 T F T <--- 21 420-439 110 109 110 204 203 204 F F F <--- 22 440-459 98 98 98 185 185 185 F F F 23 460-479 109 109 109 203 203 203 T F T <--- 24 480-499 109 109 109 203 203 203 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 106 106 106 198 198 198 F F F 27 540-559 106 106 106 199 199 199 F F F 28 560-579 110 101 110 204 188 204 F F F <--- 29 580-599 110 101 110 204 188 204 F F F <--- Max gen: 110, Max flush: 204 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 204 204 204 F F F 1 020-039 110 110 110 204 204 204 F F F 2 040-059 111 111 111 205 205 205 F F F 3 060-079 109 109 109 203 203 203 F F F 4 080-099 110 110 110 204 204 204 F F F 5 100-119 104 104 104 193 193 193 F F F 6 120-139 106 106 106 198 198 198 F F F 7 140-159 106 106 106 198 198 198 F F F 8 160-179 109 109 109 203 203 203 F F F 9 180-199 111 111 111 205 205 205 F F F 10 200-219 106 106 106 199 199 199 F F F 11 220-239 107 107 107 201 201 201 F F F 12 240-259 107 107 107 201 201 201 F F F 13 260-279 109 109 109 203 203 203 F F F 14 280-299 111 111 111 205 205 205 F F F 15 300-319 111 111 111 205 205 205 F F F 16 320-339 110 110 110 204 204 204 F F F 17 340-359 110 110 110 204 204 204 F F F 18 360-379 109 109 109 203 203 203 F F F 19 380-399 104 104 104 191 191 191 F F F 20 400-419 111 111 111 205 205 205 F F F 21 420-439 110 110 110 204 204 204 F F F 22 440-459 98 98 98 185 185 185 F F F 23 460-479 111 111 111 205 205 205 F F F 24 480-499 109 109 109 203 203 203 F F F 25 500-519 94 94 94 178 178 178 F F F 26 520-539 106 106 106 198 198 198 F F F 27 540-559 106 106 106 199 199 199 F F F 28 560-579 110 110 110 204 204 204 F F F 29 580-599 110 110 110 204 204 204 F F F Max gen: 111, Max flush: 205 Begin loop 56 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.872382342Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.89714569Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.897647884Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.897767581Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.897816558Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.897875525Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640} {"msg":"Crucible 073478db-eb78-45fe-b9c3-8628c3bbe152 has session id: e6e5e036-e7e4-4488-856d-2cec3eb49c86","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.898919211Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"Upstairs opts: Upstairs UUID: 073478db-eb78-45fe-b9c3-8628c3bbe152, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.899011133Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"Crucible stats registered with UUID: 073478db-eb78-45fe-b9c3-8628c3bbe152","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.899067735Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:28.899117373Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.899499089Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.899586292Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.899636171Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.89984782Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.899903089Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640} {"msg":"073478db-eb78-45fe-b9c3-8628c3bbe152 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.899953719Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.899999029Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"client":"0","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.900047665Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"client":"1","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.900300215Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"client":"2","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.900367248Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.900397107Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.900424892Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"ds_connection connected from Ok(127.0.0.1:45040)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.900498588Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"ds_connection connected from Ok(127.0.0.1:58692)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.900611993Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"ds_connection connected from Ok(127.0.0.1:43476)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.900648585Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.902259174Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"client":"2","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.902369642Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"client":"0","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.902888119Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.902955402Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.903946344Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"client":"1","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"[0]R flush_numbers[0..12]: [204, 204, 205, 203, 204, 193, 198, 198, 203, 205, 199, 201]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.904820735Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"downstairs","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"[0]R generation[0..12]: [110, 110, 111, 109, 110, 104, 106, 106, 109, 111, 106, 107]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.904889612Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"downstairs","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.904943749Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"downstairs","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"[1]R flush_numbers[0..12]: [204, 204, 205, 203, 204, 193, 198, 198, 203, 205, 199, 201]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.904976955Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"downstairs","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"[1]R generation[0..12]: [110, 110, 111, 109, 110, 104, 106, 106, 109, 111, 106, 107]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905051803Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"downstairs","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905112052Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"downstairs","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"[2]R flush_numbers[0..12]: [204, 204, 205, 203, 204, 193, 198, 198, 203, 205, 199, 201]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905172983Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"downstairs","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"[2]R generation[0..12]: [110, 110, 111, 109, 110, 104, 106, 106, 109, 111, 106, 107]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905231979Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"downstairs","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:13:28.905281918Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"downstairs","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905327679Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"downstairs","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905379702Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"downstairs","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905428028Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"downstairs","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905477045Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"":"downstairs","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905522615Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905557765Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"client":"0","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905611401Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"client":"1","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905665368Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"client":"2","":"downstairs","session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905714264Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905759684Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"073478db-eb78-45fe-b9c3-8628c3bbe152 is now active with session: e6e5e036-e7e4-4488-856d-2cec3eb49c86","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905813761Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905859622Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640,"session_id":"e6e5e036-e7e4-4488-856d-2cec3eb49c86"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905908018Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640} {"msg":"Activated sub_volume 073478db-eb78-45fe-b9c3-8628c3bbe152","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:28.905952236Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 516 len 20480 data: 7 7 5 6 5 02/30 Write block 403 len 32768 data: 12 14 13 13 12 12 10 8 03/30 Read block 118 len 20480 04/30 Read block 496 len 20480 05/30 Read block 541 len 36864 06/30 Read block 356 len 8192 07/30 Read block 225 len 8192 08/30 Write block 508 len 12288 data: 7 7 10 09/30 Read block 363 len 12288 10/30 Write block 277 len 36864 data: 8 9 8 8 8 7 8 11 8 11/30 Read block 491 len 36864 12/30 Read block 102 len 20480 13/30 Read block 171 len 40960 14/30 Write block 577 len 32768 data: 9 10 10 11 12 12 12 13 15/30 Write block 126 len 4096 data: 8 16/30 Read block 376 len 36864 17/30 Write block 460 len 20480 data: 14 10 12 11 10 18/30 Read block 306 len 28672 19/30 Read block 557 len 36864 20/30 Write block 124 len 32768 data: 7 7 9 10 9 11 10 8 21/30 Read block 144 len 4096 22/30 Write block 43 len 8192 data: 8 8 23/30 Write block 389 len 32768 data: 6 7 9 10 8 7 6 10 24/30 Read block 143 len 16384 25/30 Read block 343 len 4096 26/30 Write block 205 len 4096 data: 10 27/30 Write block 539 len 4096 data: 10 28/30 Read block 27 len 12288 29/30 Read block 249 len 40960 30/30 Write block 476 len 40960 data: 9 8 7 8 9 8 9 9 7 6 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Write 8 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 26 1 28 Done 29 4 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:28.939157548Z","hostname":"bmat-EVT22200007-0000ead9","pid":1640} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 204 204 204 F F F 1 020-039 110 110 110 204 204 204 F F F 2 040-059 111 111 111 205 205 205 T F T <--- 3 060-079 109 109 109 203 203 203 F F F 4 080-099 110 110 110 204 204 204 F F F 5 100-119 104 104 104 193 193 193 F F F 6 120-139 106 106 106 198 198 198 T F T <--- 7 140-159 106 106 106 198 198 198 F F F 8 160-179 109 109 109 203 203 203 F F F 9 180-199 111 111 111 205 205 205 F F F 10 200-219 106 106 106 199 199 199 T F T <--- 11 220-239 107 107 107 201 201 201 F F F 12 240-259 107 107 107 201 201 201 F F F 13 260-279 109 109 109 203 203 203 T F T <--- 14 280-299 111 111 111 205 205 205 T F T <--- 15 300-319 111 111 111 205 205 205 F F F 16 320-339 110 110 110 204 204 204 F F F 17 340-359 110 110 110 204 204 204 F F F 18 360-379 109 109 109 203 203 203 F F F 19 380-399 104 104 104 191 191 191 T F T <--- 20 400-419 111 111 111 205 205 205 T T T 21 420-439 110 110 110 204 204 204 F F F 22 440-459 98 98 98 185 185 185 F F F 23 460-479 111 111 111 205 205 205 T F T <--- 24 480-499 109 109 109 203 203 203 T F T <--- 25 500-519 94 94 94 178 178 178 T T T 26 520-539 106 106 106 198 198 198 T T T 27 540-559 106 106 106 199 199 199 F F F 28 560-579 110 110 110 204 204 204 T F T <--- 29 580-599 110 110 110 204 204 204 T F T <--- Max gen: 111, Max flush: 205 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 204 204 204 F F F 1 020-039 110 110 110 204 204 204 F F F 2 040-059 112 112 112 206 206 206 F F F 3 060-079 109 109 109 203 203 203 F F F 4 080-099 110 110 110 204 204 204 F F F 5 100-119 104 104 104 193 193 193 F F F 6 120-139 112 112 112 206 206 206 F F F 7 140-159 106 106 106 198 198 198 F F F 8 160-179 109 109 109 203 203 203 F F F 9 180-199 111 111 111 205 205 205 F F F 10 200-219 112 112 112 206 206 206 F F F 11 220-239 107 107 107 201 201 201 F F F 12 240-259 107 107 107 201 201 201 F F F 13 260-279 112 112 112 206 206 206 F F F 14 280-299 112 112 112 206 206 206 F F F 15 300-319 111 111 111 205 205 205 F F F 16 320-339 110 110 110 204 204 204 F F F 17 340-359 110 110 110 204 204 204 F F F 18 360-379 109 109 109 203 203 203 F F F 19 380-399 112 112 112 206 206 206 F F F 20 400-419 112 112 112 206 206 206 F F F 21 420-439 110 110 110 204 204 204 F F F 22 440-459 98 98 98 185 185 185 F F F 23 460-479 112 112 112 206 206 206 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 112 112 112 206 206 206 F F F 26 520-539 112 112 112 206 206 206 F F F 27 540-559 106 106 106 199 199 199 F F F 28 560-579 112 112 112 206 206 206 F F F 29 580-599 112 112 112 206 206 206 F F F Max gen: 112, Max flush: 206 Begin loop 57 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.61642584Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:33.641591695Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.641691964Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.665334082Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.665965632Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.666062253Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.666096481Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.666140669Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648} {"msg":"Crucible dfaada33-3a2c-4c91-8b74-dfeaae7da299 has session id: 3984877f-fe27-4962-b047-5974ab7292e7","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.667143354Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"Upstairs opts: Upstairs UUID: dfaada33-3a2c-4c91-8b74-dfeaae7da299, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.667227951Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"Crucible stats registered with UUID: dfaada33-3a2c-4c91-8b74-dfeaae7da299","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.667268572Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:33.66730295Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.667787449Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.667870884Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.667911835Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.668033657Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.668078305Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648} {"msg":"dfaada33-3a2c-4c91-8b74-dfeaae7da299 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.668398258Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.668452756Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"client":"0","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.668503286Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"client":"1","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.668542574Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"client":"2","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.66858106Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.668624226Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.668669435Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62839)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.669369501Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:60779)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.669479309Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:44289)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.669521021Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.671179505Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"client":"2","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.671279032Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"client":"1","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.671326436Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"client":"0","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.672214444Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.672362076Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"[0]R flush_numbers[0..12]: [204, 204, 206, 203, 204, 193, 206, 198, 203, 205, 206, 201]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.672710897Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"downstairs","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"[0]R generation[0..12]: [110, 110, 112, 109, 110, 104, 112, 106, 109, 111, 112, 107]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.672769894Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"downstairs","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.672819632Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"downstairs","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"[1]R flush_numbers[0..12]: [204, 204, 206, 203, 204, 193, 206, 198, 203, 205, 206, 201]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.672866014Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"downstairs","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"[1]R generation[0..12]: [110, 110, 112, 109, 110, 104, 112, 106, 109, 111, 112, 107]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.672912486Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"downstairs","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.672959088Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"downstairs","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"[2]R flush_numbers[0..12]: [204, 204, 206, 203, 204, 193, 206, 198, 203, 205, 206, 201]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673004588Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"downstairs","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"[2]R generation[0..12]: [110, 110, 112, 109, 110, 104, 112, 106, 109, 111, 112, 107]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673049618Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"downstairs","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673103915Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"downstairs","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"Max found gen is 113","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673150427Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"downstairs","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"Generation requested: 114 >= found:113","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673194194Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"downstairs","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.67323255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"downstairs","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673275585Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"":"downstairs","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673324572Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673365313Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"client":"0","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673405442Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"client":"1","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673471423Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"client":"2","":"downstairs","session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673521622Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.6735713Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"dfaada33-3a2c-4c91-8b74-dfeaae7da299 is now active with session: 3984877f-fe27-4962-b047-5974ab7292e7","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673618985Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673657Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648,"session_id":"3984877f-fe27-4962-b047-5974ab7292e7"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:33.673697841Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648} {"msg":"Activated sub_volume dfaada33-3a2c-4c91-8b74-dfeaae7da299","v":0,"name":"crucible","level":30,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 time":"2026-03-04T01:13:33.673739473Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 234 len 24576 data: 12 9 11 10 10 12 02/30 Read block 288 len 24576 03/30 Write block 426 len 32768 data: 10 9 9 12 12 12 8 9 04/30 Read block 410 len 16384 05/30 Write block 433 len 32768 data: 10 10 12 14 17 16 12 11 06/30 Write block 344 len 12288 data: 7 7 8 07/30 Read block 486 len 40960 08/30 Flush 09/30 Write block 458 len 12288 data: 15 15 15 10/30 Write block 503 len 28672 data: 9 9 7 6 6 8 8 11/30 Read block 461 len 24576 12/30 Write block 392 len 8192 data: 11 9 13/30 Read block 303 len 24576 14/30 Write block 11 len 40960 data: 9 8 8 7 7 5 5 5 8 9 15/30 Read block 172 len 36864 16/30 Write block 235 len 24576 data: 10 12 11 11 13 12 17/30 Write block 213 len 40960 data: 5 4 4 6 8 8 6 7 5 6 18/30 Read block 445 len 36864 19/30 Read block 487 len 28672 20/30 Write block 126 len 40960 data: 10 11 10 12 11 9 10 11 13 12 21/30 Write block 255 len 20480 data: 8 7 10 10 12 22/30 Write block 36 len 16384 data: 8 7 7 6 23/30 Read block 156 len 12288 24/30 Read block 427 len 32768 25/30 Read block 79 len 24576 26/30 Write block 328 len 20480 data: 6 5 6 5 5 27/30 Read block 552 len 40960 28/30 Read block 83 len 32768 29/30 Read block 311 len 32768 30/30 Write block 334 len 36864 data: 6 5 8 9 8 8 9 11 10 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Read 4 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1007)) Some(JobId(1007)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:33.711899475Z","hostname":"bmat-EVT22200007-0000ead9","pid":1648} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 110 110 110 204 204 204 F T T <--- 1 020-039 110 110 110 204 204 204 F T T <--- 2 040-059 112 112 112 206 206 206 F F F 3 060-079 109 109 109 203 203 203 F F F 4 080-099 110 110 110 204 204 204 F F F 5 100-119 104 104 104 193 193 193 F F F 6 120-139 112 112 112 206 206 206 F T T <--- 7 140-159 106 106 106 198 198 198 F F F 8 160-179 109 109 109 203 203 203 F F F 9 180-199 111 111 111 205 205 205 F F F 10 200-219 112 112 112 206 206 206 F T T <--- 11 220-239 107 114 114 201 207 207 T T T <--- 12 240-259 107 107 107 201 201 201 F T T <--- 13 260-279 112 112 112 206 206 206 F F F 14 280-299 112 112 112 206 206 206 F F F 15 300-319 111 111 111 205 205 205 F F F 16 320-339 110 110 110 204 204 204 F T T <--- 17 340-359 110 114 114 204 207 207 F T T <--- 18 360-379 109 109 109 203 203 203 F F F 19 380-399 112 112 112 206 206 206 F T T <--- 20 400-419 112 112 112 206 206 206 F F F 21 420-439 110 114 114 204 207 207 F F F <--- 22 440-459 98 114 114 185 207 207 F T T <--- 23 460-479 112 112 112 206 206 206 F T T <--- 24 480-499 112 112 112 206 206 206 F F F 25 500-519 112 112 112 206 206 206 F T T <--- 26 520-539 112 112 112 206 206 206 F F F 27 540-559 106 106 106 199 199 199 F F F 28 560-579 112 112 112 206 206 206 F F F 29 580-599 112 112 112 206 206 206 F F F Max gen: 114, Max flush: 207 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 208 208 208 F F F 1 020-039 115 115 115 208 208 208 F F F 2 040-059 112 112 112 206 206 206 F F F 3 060-079 109 109 109 203 203 203 F F F 4 080-099 110 110 110 204 204 204 F F F 5 100-119 104 104 104 193 193 193 F F F 6 120-139 115 115 115 208 208 208 F F F 7 140-159 106 106 106 198 198 198 F F F 8 160-179 109 109 109 203 203 203 F F F 9 180-199 111 111 111 205 205 205 F F F 10 200-219 115 115 115 208 208 208 F F F 11 220-239 115 115 115 208 208 208 F F F 12 240-259 115 115 115 208 208 208 F F F 13 260-279 112 112 112 206 206 206 F F F 14 280-299 112 112 112 206 206 206 F F F 15 300-319 111 111 111 205 205 205 F F F 16 320-339 115 115 115 208 208 208 F F F 17 340-359 115 115 115 208 208 208 F F F 18 360-379 109 109 109 203 203 203 F F F 19 380-399 115 115 115 208 208 208 F F F 20 400-419 112 112 112 206 206 206 F F F 21 420-439 114 114 114 207 207 207 F F F 22 440-459 115 115 115 208 208 208 F F F 23 460-479 115 115 115 208 208 208 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 115 115 115 208 208 208 F F F 26 520-539 112 112 112 206 206 206 F F F 27 540-559 106 106 106 199 199 199 F F F 28 560-579 112 112 112 206 206 206 F F F 29 580-599 112 112 112 206 206 206 F F F Max gen: 115, Max flush: 208 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.393576967Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.422622464Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.42310066Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.423170348Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.423192973Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.423225688Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656} {"msg":"Crucible cbe7f285-2c2f-4da2-8a37-c972b0896f1a has session id: 70cf1d6b-60cb-484c-8f5d-fb368f18bcb6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.424386165Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Upstairs opts: Upstairs UUID: cbe7f285-2c2f-4da2-8a37-c972b0896f1a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.424446535Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Crucible stats registered with UUID: cbe7f285-2c2f-4da2-8a37-c972b0896f1a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.424481795Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:38.424505662Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.424821678Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.424880023Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.424906546Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.425088666Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.425119738Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656} {"msg":"cbe7f285-2c2f-4da2-8a37-c972b0896f1a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.425223383Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.425253232Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"client":"0","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.425278502Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"client":"1","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.425303612Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"client":"2","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.42532776Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.425399662Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:44761)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.425531373Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:44776)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.425566071Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.425593786Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:41254)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.425763733Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.42757026Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"client":"1","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.427839603Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"client":"0","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.42794438Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"client":"2","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.428231328Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.428384121Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"[0]R flush_numbers[0..12]: [208, 208, 206, 203, 204, 193, 208, 198, 203, 205, 208, 208]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.428861155Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"downstairs","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"[0]R generation[0..12]: [115, 115, 112, 109, 110, 104, 115, 106, 109, 111, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.428911575Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"downstairs","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.428956173Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"downstairs","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"[1]R flush_numbers[0..12]: [208, 208, 206, 203, 204, 193, 208, 198, 203, 205, 208, 208]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.428999179Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"downstairs","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"[1]R generation[0..12]: [115, 115, 112, 109, 110, 104, 115, 106, 109, 111, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429041953Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"downstairs","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429086211Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"downstairs","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"[2]R flush_numbers[0..12]: [208, 208, 206, 203, 204, 193, 208, 198, 203, 205, 208, 208]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429128755Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"downstairs","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"[2]R generation[0..12]: [115, 115, 112, 109, 110, 104, 115, 106, 109, 111, 115, 115]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.42917183Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"downstairs","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.4292272Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"downstairs","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429263813Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"downstairs","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429301567Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429346546Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"downstairs","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429382588Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"":"downstairs","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429420162Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429454009Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"client":"0","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429491914Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"client":"1","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429539458Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"client":"2","":"downstairs","session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429596231Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429632613Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 cbe7f285-2c2f-4da2-8a37-c972b0896f1a is now active with session: 70cf1d6b-60cb-484c-8f5d-fb368f18bcb6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429673524Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429708623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656,"session_id":"70cf1d6b-60cb-484c-8f5d-fb368f18bcb6"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429745135Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656} {"msg":"Activated sub_volume cbe7f285-2c2f-4da2-8a37-c972b0896f1a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:38.429776748Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 283 len 16384 data: 9 12 9 10 02/30 Read block 119 len 32768 03/30 Read block 427 len 16384 04/30 Read block 385 len 32768 05/30 Write block 429 len 12288 data: 13 13 13 06/30 Read block 341 len 20480 07/30 Write block 205 len 12288 data: 11 12 13 08/30 Write block 212 len 40960 data: 8 6 5 5 7 9 9 7 8 6 09/30 Write block 72 len 24576 data: 7 8 7 7 6 5 10/30 Write block 465 len 24576 data: 9 11 10 7 7 8 11/30 Read block 476 len 24576 12/30 Read block 185 len 4096 13/30 Read block 309 len 4096 14/30 Read block 170 len 8192 15/30 Write block 567 len 8192 data: 10 10 16/30 Write block 567 len 16384 data: 11 11 10 13 17/30 Write block 334 len 40960 data: 7 6 9 10 9 9 10 12 11 7 18/30 Read block 529 len 28672 19/30 Read block 419 len 4096 20/30 Flush 21/30 Write block 367 len 36864 data: 12 11 9 10 9 9 9 9 5 22/30 Read block 7 len 4096 23/30 Write block 151 len 4096 data: 8 24/30 Read block 537 len 20480 25/30 Write block 316 len 20480 data: 9 8 7 8 13 26/30 Read block 229 len 16384 27/30 Write block 346 len 36864 data: 9 8 7 7 6 6 4 5 3 28/30 Write block 599 len 4096 data: 2 29/30 Read block 222 len 8192 30/30 Write block 374 len 28672 data: 10 6 6 7 8 9 8 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 4 Done Done Sent false 1003 Acked Read 8 Done Done Sent false 1004 Acked Write 3 Done Done Sent false 1005 Acked Read 5 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:38.460794329Z","hostname":"bmat-EVT22200007-0000ead9","pid":1656} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 208 208 208 F F F 1 020-039 115 115 115 208 208 208 F F F 2 040-059 112 112 112 206 206 206 F F F 3 060-079 116 116 109 209 209 203 F F F <--- 4 080-099 110 110 110 204 204 204 F F F 5 100-119 104 104 104 193 193 193 F F F 6 120-139 115 115 115 208 208 208 F F F 7 140-159 106 106 106 198 198 198 T T F <--- 8 160-179 109 109 109 203 203 203 F F F 9 180-199 111 111 111 205 205 205 F F F 10 200-219 116 116 115 209 209 208 F F F <--- 11 220-239 116 116 115 209 209 208 F F F <--- 12 240-259 115 115 115 208 208 208 F F F 13 260-279 112 112 112 206 206 206 F F F 14 280-299 116 116 112 209 209 206 F F T <--- 15 300-319 111 111 111 205 205 205 T T F <--- 16 320-339 116 116 115 209 209 208 T T F <--- 17 340-359 116 116 115 209 209 208 T T F <--- 18 360-379 109 109 109 203 203 203 T T F <--- 19 380-399 115 115 115 208 208 208 T T F <--- 20 400-419 112 112 112 206 206 206 F F F 21 420-439 116 116 114 209 209 207 F F F <--- 22 440-459 115 115 115 208 208 208 F F F 23 460-479 116 116 115 209 209 208 F F F <--- 24 480-499 112 112 112 206 206 206 F F F 25 500-519 115 115 115 208 208 208 F F F 26 520-539 112 112 112 206 206 206 F F F 27 540-559 106 106 106 199 199 199 F F F 28 560-579 116 116 112 209 209 206 F F F <--- 29 580-599 112 112 112 206 206 206 T T F <--- Max gen: 116, Max flush: 209 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 208 208 208 F F F 1 020-039 115 115 115 208 208 208 F F F 2 040-059 112 112 112 206 206 206 F F F 3 060-079 116 116 116 209 209 209 F F F 4 080-099 110 110 110 204 204 204 F F F 5 100-119 104 104 104 193 193 193 F F F 6 120-139 115 115 115 208 208 208 F F F 7 140-159 117 117 117 210 210 210 F F F 8 160-179 109 109 109 203 203 203 F F F 9 180-199 111 111 111 205 205 205 F F F 10 200-219 116 116 116 209 209 209 F F F 11 220-239 116 116 116 209 209 209 F F F 12 240-259 115 115 115 208 208 208 F F F 13 260-279 112 112 112 206 206 206 F F F 14 280-299 116 116 116 209 209 209 F F F 15 300-319 117 117 117 210 210 210 F F F 16 320-339 117 117 117 210 210 210 F F F 17 340-359 117 117 117 210 210 210 F F F 18 360-379 117 117 117 210 210 210 F F F 19 380-399 117 117 117 210 210 210 F F F 20 400-419 112 112 112 206 206 206 F F F 21 420-439 116 116 116 209 209 209 F F F 22 440-459 115 115 115 208 208 208 F F F 23 460-479 116 116 116 209 209 209 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 115 115 115 208 208 208 F F F 26 520-539 112 112 112 206 206 206 F F F 27 540-559 106 106 106 199 199 199 F F F 28 560-579 116 116 116 209 209 209 F F F 29 580-599 117 117 117 210 210 210 F F F Max gen: 117, Max flush: 210 Begin loop 59 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.098246776Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.127684328Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.128247102Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.128371168Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.128416638Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.128486576Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664} {"msg":"Crucible 4923e82b-b94c-4e25-a5f3-bac8aadc30f4 has session id: a5328af1-de67-4490-b471-adf9a629fe24","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.129268805Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"Upstairs opts: Upstairs UUID: 4923e82b-b94c-4e25-a5f3-bac8aadc30f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.129366799Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"Crucible stats registered with UUID: 4923e82b-b94c-4e25-a5f3-bac8aadc30f4","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.129422359Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:53.129471216Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.12982175Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.129893221Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.12994808Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.130072677Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.130120752Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664} {"msg":"4923e82b-b94c-4e25-a5f3-bac8aadc30f4 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.13017044Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.130222333Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"client":"0","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.130275017Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"client":"1","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.130327381Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"client":"2","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.130378863Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.130647094Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.13074107Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"ds_connection connected from Ok(127.0.0.1:48047)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.130803814Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"ds_connection connected from Ok(127.0.0.1:50824)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.130834204Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"ds_connection connected from Ok(127.0.0.1:62661)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.130862931Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.132809114Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"client":"0","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.133009922Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"client":"1","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.133371809Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"client":"2","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.133611784Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.133749276Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"[0]R flush_numbers[0..12]: [208, 208, 206, 209, 204, 193, 208, 210, 203, 205, 209, 209]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134211872Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"downstairs","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"[0]R generation[0..12]: [115, 115, 112, 116, 110, 104, 115, 117, 109, 111, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134255678Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"downstairs","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134288303Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"downstairs","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"[1]R flush_numbers[0..12]: [208, 208, 206, 209, 204, 193, 208, 210, 203, 205, 209, 209]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134319284Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"downstairs","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"[1]R generation[0..12]: [115, 115, 112, 116, 110, 104, 115, 117, 109, 111, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134350316Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"downstairs","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134382259Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"downstairs","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"[2]R flush_numbers[0..12]: [208, 208, 206, 209, 204, 193, 208, 210, 203, 205, 209, 209]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134413551Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"downstairs","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"[2]R generation[0..12]: [115, 115, 112, 116, 110, 104, 115, 117, 109, 111, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134443931Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"downstairs","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134475674Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"downstairs","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134502146Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"downstairs","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134529581Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"downstairs","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134555232Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"downstairs","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"All extents match","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2026-03-04T01:13:53.134591794Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"":"downstairs","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.13461998Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134644488Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"client":"0","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.13467551Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"client":"1","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134702523Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"client":"2","":"downstairs","session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134730218Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134755638Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"4923e82b-b94c-4e25-a5f3-bac8aadc30f4 is now active with session: a5328af1-de67-4490-b471-adf9a629fe24","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134784816Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134809104Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664,"session_id":"a5328af1-de67-4490-b471-adf9a629fe24"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134835236Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664} {"msg":"Activated sub_volume 4923e82b-b94c-4e25-a5f3-bac8aadc30f4","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:53.134857059Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 329 len 36864 data: 6 7 6 6 5 8 7 10 11 02/30 Read block 389 len 40960 03/30 Read block 379 len 36864 04/30 Write block 96 len 4096 data: 13 05/30 Flush 06/30 Write block 289 len 20480 data: 9 10 13 12 7 07/30 Write block 342 len 4096 data: 12 08/30 Write block 302 len 8192 data: 9 10 09/30 Read block 151 len 4096 10/30 Read block 227 len 32768 11/30 Read block 553 len 24576 12/30 Write block 433 len 4096 data: 11 13/30 Read block 17 len 12288 14/30 Read block 529 len 4096 15/30 Read block 76 len 12288 16/30 Write block 470 len 4096 data: 9 17/30 Flush 18/30 Write block 231 len 24576 data: 10 10 11 13 11 13 19/30 Write block 544 len 8192 data: 7 8 20/30 Read block 407 len 40960 21/30 Read block 336 len 32768 22/30 Write block 448 len 8192 data: 9 10 23/30 Write block 143 len 20480 data: 9 9 8 10 12 24/30 Write block 580 len 28672 data: 12 13 13 13 14 11 10 25/30 Write block 274 len 20480 data: 10 9 10 9 10 26/30 Read block 45 len 28672 27/30 Write block 386 len 40960 data: 8 7 7 7 8 10 12 10 8 7 28/30 Read block 195 len 36864 29/30 Read block 565 len 32768 30/30 Write block 40 len 24576 data: 5 5 5 9 9 11 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 5 Done Done Sent false 1006 Acked Write 1 Done Done Sent false 1007 Acked Write 2 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Read 1 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 25 27 Done 24 24 0 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:53.169176737Z","hostname":"bmat-EVT22200007-0000ead9","pid":1664} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 208 208 208 F F F 1 020-039 115 115 115 208 208 208 F F F 2 040-059 112 112 112 206 206 206 T T F <--- 3 060-079 116 116 116 209 209 209 F F F 4 080-099 118 118 118 211 211 211 F F F 5 100-119 104 104 104 193 193 193 F F F 6 120-139 115 115 115 208 208 208 F F F 7 140-159 117 117 117 210 210 210 T T F <--- 8 160-179 109 109 109 203 203 203 F F F 9 180-199 111 111 111 205 205 205 F F F 10 200-219 116 116 116 209 209 209 F F F 11 220-239 116 116 116 209 209 209 T T F <--- 12 240-259 115 115 115 208 208 208 F F F 13 260-279 112 112 112 206 206 206 T T F <--- 14 280-299 118 118 116 212 212 209 F F F <--- 15 300-319 118 118 117 212 212 210 F F F <--- 16 320-339 118 118 118 211 211 211 F F F 17 340-359 118 118 117 212 212 210 F F F <--- 18 360-379 117 117 117 210 210 210 F F F 19 380-399 117 117 117 210 210 210 T T F <--- 20 400-419 112 112 112 206 206 206 F F F 21 420-439 118 118 116 212 212 209 F F F <--- 22 440-459 115 115 115 208 208 208 T T F <--- 23 460-479 118 118 116 212 212 209 F F F <--- 24 480-499 112 112 112 206 206 206 F F F 25 500-519 115 115 115 208 208 208 F F F 26 520-539 112 112 112 206 206 206 F F F 27 540-559 106 106 106 199 199 199 T T F <--- 28 560-579 116 116 116 209 209 209 F F F 29 580-599 117 117 117 210 210 210 T T F <--- Max gen: 118, Max flush: 212 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 208 208 208 F F F 1 020-039 115 115 115 208 208 208 F F F 2 040-059 119 119 119 213 213 213 F F F 3 060-079 116 116 116 209 209 209 F F F 4 080-099 118 118 118 211 211 211 F F F 5 100-119 104 104 104 193 193 193 F F F 6 120-139 115 115 115 208 208 208 F F F 7 140-159 119 119 119 213 213 213 F F F 8 160-179 109 109 109 203 203 203 F F F 9 180-199 111 111 111 205 205 205 F F F 10 200-219 116 116 116 209 209 209 F F F 11 220-239 119 119 119 213 213 213 F F F 12 240-259 115 115 115 208 208 208 F F F 13 260-279 119 119 119 213 213 213 F F F 14 280-299 118 118 118 212 212 212 F F F 15 300-319 118 118 118 212 212 212 F F F 16 320-339 118 118 118 211 211 211 F F F 17 340-359 118 118 118 212 212 212 F F F 18 360-379 117 117 117 210 210 210 F F F 19 380-399 119 119 119 213 213 213 F F F 20 400-419 112 112 112 206 206 206 F F F 21 420-439 118 118 118 212 212 212 F F F 22 440-459 119 119 119 213 213 213 F F F 23 460-479 118 118 118 212 212 212 F F F 24 480-499 112 112 112 206 206 206 F F F 25 500-519 115 115 115 208 208 208 F F F 26 520-539 112 112 112 206 206 206 F F F 27 540-559 119 119 119 213 213 213 F F F 28 560-579 116 116 116 209 209 209 F F F 29 580-599 119 119 119 213 213 213 F F F Max gen: 119, Max flush: 213 Begin loop 60 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.803558351Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.824428841Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.824975794Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.825103507Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.825131392Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.825167063Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672} {"msg":"Crucible ae2f316b-3cf4-42ec-bac4-8b3bdc7c77bd has session id: 8f7535fc-8047-4635-a9de-cac49a2a5353","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.826491986Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"Upstairs opts: Upstairs UUID: ae2f316b-3cf4-42ec-bac4-8b3bdc7c77bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.826551554Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"Crucible stats registered with UUID: ae2f316b-3cf4-42ec-bac4-8b3bdc7c77bd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.826587405Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:57.826612915Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.827071734Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.827126382Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.827277191Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.827530502Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.827571533Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672} {"msg":"ae2f316b-3cf4-42ec-bac4-8b3bdc7c77bd active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.827800537Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.827873131Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"client":"0","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.827904563Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"client":"1","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.827931566Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"client":"2","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.827969541Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.828012917Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.828051644Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"ds_connection connected from Ok(127.0.0.1:50635)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.828083918Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"ds_connection connected from Ok(127.0.0.1:57774)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.828354213Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"ds_connection connected from Ok(127.0.0.1:53733)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.828881938Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.830197392Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"client":"0","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.830270086Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"client":"2","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.830406246Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"client":"1","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.830859734Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.830925534Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"[0]R flush_numbers[0..12]: [208, 208, 213, 209, 211, 193, 208, 213, 203, 205, 209, 213]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.831234365Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"downstairs","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"[0]R generation[0..12]: [115, 115, 119, 116, 118, 104, 115, 119, 109, 111, 116, 119]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.83130754Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"downstairs","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.831370274Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"downstairs","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"[1]R flush_numbers[0..12]: [208, 208, 213, 209, 211, 193, 208, 213, 203, 205, 209, 213]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.831432357Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"downstairs","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"[1]R generation[0..12]: [115, 115, 119, 116, 118, 104, 115, 119, 109, 111, 116, 119]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.831487847Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"downstairs","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.831536023Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"downstairs","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"[2]R flush_numbers[0..12]: [208, 208, 213, 209, 211, 193, 208, 213, 203, 205, 209, 213]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.831590911Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"downstairs","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"[2]R generation[0..12]: [115, 115, 119, 116, 118, 104, 115, 119, 109, 111, 116, 119]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.83164064Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"downstairs","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2026-03-04T01:13:57.831710097Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"downstairs","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.831783603Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"downstairs","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.831823111Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"downstairs","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.831860415Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"downstairs","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.831897168Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"":"downstairs","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.831931115Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.831968198Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"client":"0","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.832004901Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"client":"1","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.832040451Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"client":"2","":"downstairs","session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.83208Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.8321258Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"ae2f316b-3cf4-42ec-bac4-8b3bdc7c77bd is now active with session: 8f7535fc-8047-4635-a9de-cac49a2a5353","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.832170188Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.832199927Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672,"session_id":"8f7535fc-8047-4635-a9de-cac49a2a5353"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.83223172Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672} {"msg":"Activated sub_volume ae2f316b-3cf4-42ec-bac4-8b3bdc7c77bd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:13:57.832258774Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 489 len 12288 data: 5 6 5 02/30 Read block 302 len 32768 03/30 Write block 389 len 24576 data: 8 9 11 13 11 9 04/30 Read block 572 len 40960 05/30 Write block 539 len 36864 data: 11 8 9 11 10 8 9 7 9 06/30 Flush 07/30 Flush 08/30 Write block 548 len 4096 data: 8 09/30 Write block 71 len 8192 data: 7 8 10/30 Read block 61 len 16384 11/30 Flush 12/30 Write block 333 len 40960 data: 6 9 8 11 12 10 10 11 13 13 13/30 Read block 129 len 16384 14/30 Flush 15/30 Write block 39 len 32768 data: 7 6 6 6 10 10 12 9 16/30 Flush 17/30 Write block 68 len 40960 data: 8 9 10 8 9 9 8 8 7 6 18/30 Read block 80 len 16384 19/30 Read block 361 len 32768 20/30 Read block 50 len 28672 21/30 Read block 63 len 20480 22/30 Read block 69 len 4096 23/30 Read block 299 len 36864 24/30 Write block 574 len 32768 data: 9 10 8 10 11 11 13 14 25/30 Write block 518 len 24576 data: 6 7 6 5 8 7 26/30 Flush 27/30 Write block 110 len 36864 data: 8 8 8 14 13 11 12 14 13 28/30 Read block 422 len 16384 29/30 Write block 284 len 8192 data: 13 10 30/30 Write block 342 len 16384 data: 14 8 8 8 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Sent Done false 1001 Acked Read 8 Done Sent Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Write 6 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 2 33 Done 29 0 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:13:57.870534093Z","hostname":"bmat-EVT22200007-0000ead9","pid":1672} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 208 208 208 F F F 1 020-039 120 115 120 218 208 218 F F F <--- 2 040-059 120 119 120 218 213 218 F F F <--- 3 060-079 120 116 120 219 209 219 F F F <--- 4 080-099 118 118 118 211 211 211 F F F 5 100-119 104 104 104 193 193 193 T F T <--- 6 120-139 115 115 115 208 208 208 F F F 7 140-159 119 119 119 213 213 213 F F F 8 160-179 109 109 109 203 203 203 F F F 9 180-199 111 111 111 205 205 205 F F F 10 200-219 116 116 116 209 209 209 F F F 11 220-239 119 119 119 213 213 213 F F F 12 240-259 115 115 115 208 208 208 F F F 13 260-279 119 119 119 213 213 213 F F F 14 280-299 118 118 118 212 212 212 T F T <--- 15 300-319 118 118 118 212 212 212 F F F 16 320-339 120 118 120 217 211 217 F F F <--- 17 340-359 120 118 120 217 212 217 T F T <--- 18 360-379 117 117 117 210 210 210 F F F 19 380-399 120 119 120 214 213 214 F F F <--- 20 400-419 112 112 112 206 206 206 F F F 21 420-439 118 118 118 212 212 212 F F F 22 440-459 119 119 119 213 213 213 F F F 23 460-479 118 118 118 212 212 212 F F F 24 480-499 120 112 120 214 206 214 F F F <--- 25 500-519 120 115 120 219 208 219 F F F <--- 26 520-539 120 112 120 219 206 219 F F F <--- 27 540-559 120 119 120 216 213 216 F F F <--- 28 560-579 120 116 120 219 209 219 F F F <--- 29 580-599 120 119 120 219 213 219 F F F <--- Max gen: 120, Max flush: 219 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 208 208 208 F F F 1 020-039 120 120 120 218 218 218 F F F 2 040-059 120 120 120 218 218 218 F F F 3 060-079 120 120 120 219 219 219 F F F 4 080-099 118 118 118 211 211 211 F F F 5 100-119 121 121 121 220 220 220 F F F 6 120-139 115 115 115 208 208 208 F F F 7 140-159 119 119 119 213 213 213 F F F 8 160-179 109 109 109 203 203 203 F F F 9 180-199 111 111 111 205 205 205 F F F 10 200-219 116 116 116 209 209 209 F F F 11 220-239 119 119 119 213 213 213 F F F 12 240-259 115 115 115 208 208 208 F F F 13 260-279 119 119 119 213 213 213 F F F 14 280-299 121 121 121 220 220 220 F F F 15 300-319 118 118 118 212 212 212 F F F 16 320-339 120 120 120 217 217 217 F F F 17 340-359 121 121 121 220 220 220 F F F 18 360-379 117 117 117 210 210 210 F F F 19 380-399 120 120 120 214 214 214 F F F 20 400-419 112 112 112 206 206 206 F F F 21 420-439 118 118 118 212 212 212 F F F 22 440-459 119 119 119 213 213 213 F F F 23 460-479 118 118 118 212 212 212 F F F 24 480-499 120 120 120 214 214 214 F F F 25 500-519 120 120 120 219 219 219 F F F 26 520-539 120 120 120 219 219 219 F F F 27 540-559 120 120 120 216 216 216 F F F 28 560-579 120 120 120 219 219 219 F F F 29 580-599 120 120 120 219 219 219 F F F Max gen: 121, Max flush: 220 Begin loop 61 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.523517856Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.549392184Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.549936592Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.550049656Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.550074555Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.55010722Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680} {"msg":"Crucible 566d832b-5596-427a-9084-731182990ecc has session id: 35cbd705-93de-4dde-8a41-b6e4b43e7f89","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.551027522Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"Upstairs opts: Upstairs UUID: 566d832b-5596-427a-9084-731182990ecc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.551082861Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"Crucible stats registered with UUID: 566d832b-5596-427a-9084-731182990ecc","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.551109684Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:02.55113255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.551551409Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.551596789Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.5516228Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.551832706Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.551863026Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680} {"msg":"566d832b-5596-427a-9084-731182990ecc active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.551968655Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.552016159Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"client":"0","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.552048142Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"client":"1","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.552072911Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"client":"2","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.55209748Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.55212261Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.552214031Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"ds_connection connected from Ok(127.0.0.1:39352)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.552412103Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"ds_connection connected from Ok(127.0.0.1:41301)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.552628301Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"ds_connection connected from Ok(127.0.0.1:40116)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.552749371Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.553897975Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"client":"2","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.554145074Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"client":"0","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.55466333Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"client":"1","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.554718078Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555123882Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"[0]R flush_numbers[0..12]: [208, 218, 218, 219, 211, 220, 208, 213, 203, 205, 209, 213]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555425038Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"downstairs","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"[0]R generation[0..12]: [115, 120, 120, 120, 118, 121, 115, 119, 109, 111, 116, 119]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555462091Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"downstairs","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555499756Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"downstairs","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"[1]R flush_numbers[0..12]: [208, 218, 218, 219, 211, 220, 208, 213, 203, 205, 209, 213]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555528753Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"downstairs","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"[1]R generation[0..12]: [115, 120, 120, 120, 118, 121, 115, 119, 109, 111, 116, 119]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555556559Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"downstairs","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555585937Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"downstairs","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"[2]R flush_numbers[0..12]: [208, 218, 218, 219, 211, 220, 208, 213, 203, 205, 209, 213]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555614323Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"downstairs","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"[2]R generation[0..12]: [115, 120, 120, 120, 118, 121, 115, 119, 109, 111, 116, 119]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555641928Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"downstairs","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555670735Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"downstairs","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555694792Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555731415Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"downstairs","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555755633Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"downstairs","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555779209Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"":"downstairs","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555803958Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555826794Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"client":"0","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555851743Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"client":"1","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555882644Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"client":"2","":"downstairs","session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555908485Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555932132Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"566d832b-5596-427a-9084-731182990ecc is now active with session: 35cbd705-93de-4dde-8a41-b6e4b43e7f89","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555958805Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.555981069Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680,"session_id":"35cbd705-93de-4dde-8a41-b6e4b43e7f89"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.556004896Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680} {"msg":"Activated sub_volume 566d832b-5596-427a-9084-731182990ecc","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:02.556025246Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 270 len 16384 data: 11 11 11 10 02/30 Write block 179 len 40960 data: 10 13 9 9 9 9 10 10 11 12 03/30 Read block 597 len 12288 04/30 Read block 504 len 8192 05/30 Read block 486 len 24576 06/30 Read block 212 len 24576 07/30 Write block 322 len 12288 data: 14 14 12 08/30 Write block 208 len 20480 data: 13 13 11 9 9 09/30 Read block 16 len 20480 10/30 Read block 130 len 16384 11/30 Write block 45 len 20480 data: 13 10 9 7 9 12/30 Read block 537 len 4096 13/30 Write block 196 len 20480 data: 12 10 11 9 11 14/30 Read block 75 len 4096 15/30 Read block 508 len 40960 16/30 Read block 185 len 32768 17/30 Write block 291 len 28672 data: 14 13 8 9 10 9 9 18/30 Write block 343 len 4096 data: 9 19/30 Flush 20/30 Write block 67 len 36864 data: 5 9 10 11 9 10 10 9 9 21/30 Write block 469 len 12288 data: 8 10 9 22/30 Write block 212 len 40960 data: 10 7 6 6 8 10 10 8 9 7 23/30 Read block 363 len 12288 24/30 Read block 517 len 40960 25/30 Read block 98 len 4096 26/30 Read block 89 len 16384 27/30 Write block 176 len 20480 data: 13 13 9 11 14 28/30 Read block 136 len 8192 29/30 Write block 287 len 12288 data: 10 11 10 30/30 Read block 347 len 36864 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 9 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Write 3 Done Sent Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 22 1 23 Done 30 8 29 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:02.588096844Z","hostname":"bmat-EVT22200007-0000ead9","pid":1680} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 208 208 208 F F F 1 020-039 120 120 120 218 218 218 F F F 2 040-059 122 120 122 221 218 221 F F F <--- 3 060-079 120 120 120 219 219 219 T F T <--- 4 080-099 118 118 118 211 211 211 F F F 5 100-119 121 121 121 220 220 220 F F F 6 120-139 115 115 115 208 208 208 F F F 7 140-159 119 119 119 213 213 213 F F F 8 160-179 122 109 122 221 203 221 T T T <--- 9 180-199 122 111 122 221 205 221 T T T <--- 10 200-219 122 116 122 221 209 221 T T T <--- 11 220-239 119 119 119 213 213 213 T F T <--- 12 240-259 115 115 115 208 208 208 F F F 13 260-279 122 119 122 221 213 221 F T F <--- 14 280-299 122 121 122 221 220 221 T F T <--- 15 300-319 118 118 118 212 212 212 F F F 16 320-339 122 120 122 221 217 221 F T F <--- 17 340-359 122 121 122 221 220 221 F F F <--- 18 360-379 117 117 117 210 210 210 F F F 19 380-399 120 120 120 214 214 214 F F F 20 400-419 112 112 112 206 206 206 F F F 21 420-439 118 118 118 212 212 212 F F F 22 440-459 119 119 119 213 213 213 F F F 23 460-479 118 118 118 212 212 212 T F T <--- 24 480-499 120 120 120 214 214 214 F F F 25 500-519 120 120 120 219 219 219 F F F 26 520-539 120 120 120 219 219 219 F F F 27 540-559 120 120 120 216 216 216 F F F 28 560-579 120 120 120 219 219 219 F F F 29 580-599 120 120 120 219 219 219 F F F Max gen: 122, Max flush: 221 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 208 208 208 F F F 1 020-039 120 120 120 218 218 218 F F F 2 040-059 122 122 122 221 221 221 F F F 3 060-079 123 123 123 222 222 222 F F F 4 080-099 118 118 118 211 211 211 F F F 5 100-119 121 121 121 220 220 220 F F F 6 120-139 115 115 115 208 208 208 F F F 7 140-159 119 119 119 213 213 213 F F F 8 160-179 123 123 123 222 222 222 F F F 9 180-199 123 123 123 222 222 222 F F F 10 200-219 123 123 123 222 222 222 F F F 11 220-239 123 123 123 222 222 222 F F F 12 240-259 115 115 115 208 208 208 F F F 13 260-279 122 122 122 221 221 221 F F F 14 280-299 123 123 123 222 222 222 F F F 15 300-319 118 118 118 212 212 212 F F F 16 320-339 122 122 122 221 221 221 F F F 17 340-359 122 122 122 221 221 221 F F F 18 360-379 117 117 117 210 210 210 F F F 19 380-399 120 120 120 214 214 214 F F F 20 400-419 112 112 112 206 206 206 F F F 21 420-439 118 118 118 212 212 212 F F F 22 440-459 119 119 119 213 213 213 F F F 23 460-479 123 123 123 222 222 222 F F F 24 480-499 120 120 120 214 214 214 F F F 25 500-519 120 120 120 219 219 219 F F F 26 520-539 120 120 120 219 219 219 F F F 27 540-559 120 120 120 216 216 216 F F F 28 560-579 120 120 120 219 219 219 F F F 29 580-599 120 120 120 219 219 219 F F F Max gen: 123, Max flush: 222 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.230163492Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.25480054Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.255358124Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.255454715Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.255493401Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.255551687Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688} {"msg":"Crucible af4d24fc-9058-4b47-bab7-bc7431a69e07 has session id: ac2f4a88-0b00-4a24-96ad-f05cdf18b57f","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.256720371Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"Upstairs opts: Upstairs UUID: af4d24fc-9058-4b47-bab7-bc7431a69e07, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.256778937Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"Crucible stats registered with UUID: af4d24fc-9058-4b47-bab7-bc7431a69e07","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.256815128Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:07.256841561Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.25732117Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.257429144Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.257459384Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.257680772Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.257716182Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688} {"msg":"af4d24fc-9058-4b47-bab7-bc7431a69e07 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.257965095Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.257992569Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"client":"0","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.25801832Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"client":"1","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.258043851Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"client":"2","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.258146003Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.258176844Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.25852805Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48748)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.258676023Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33531)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.258724949Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:07.259084842Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:07.259150883Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:07.259190301Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.259215561Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.259260119Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.260117406Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"client":"2","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.260401449Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"client":"0","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.261187916Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:07.261352251Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.259876734Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60955)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.26012198Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.262160776Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"client":"1","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"[0]R flush_numbers[0..12]: [208, 218, 221, 222, 211, 220, 208, 213, 222, 222, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263341243Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"downstairs","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"[0]R generation[0..12]: [115, 120, 122, 123, 118, 121, 115, 119, 123, 123, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263532522Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"downstairs","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263572251Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"downstairs","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"[1]R flush_numbers[0..12]: [208, 218, 221, 222, 211, 220, 208, 213, 222, 222, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263611308Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"downstairs","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"[1]R generation[0..12]: [115, 120, 122, 123, 118, 121, 115, 119, 123, 123, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263640686Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"downstairs","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263670165Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"downstairs","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"[2]R flush_numbers[0..12]: [208, 218, 221, 222, 211, 220, 208, 213, 222, 222, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263698721Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"downstairs","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"[2]R generation[0..12]: [115, 120, 122, 123, 118, 121, 115, 119, 123, 123, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263727378Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"downstairs","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263757227Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"downstairs","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263781184Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"downstairs","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263806324Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"downstairs","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263830021Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"downstairs","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.26385463Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"":"downstairs","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263879669Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263902364Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"client":"0","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263927705Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"client":"1","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263952664Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"client":"2","":"downstairs","session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.263978114Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.264005218Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"af4d24fc-9058-4b47-bab7-bc7431a69e07 is now active with session: ac2f4a88-0b00-4a24-96ad-f05cdf18b57f","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.264032742Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.264055277Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688,"session_id":"ac2f4a88-0b00-4a24-96ad-f05cdf18b57f"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:17.264079034Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688} {"msg":"Activated sub_volume af4d24fc-9058-4b47-bab7-bc7431a69e07","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:14:17.264099565Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 63 len 8192 data: 6 9 02/30 Write block 251 len 32768 data: 6 7 7 8 9 8 11 11 03/30 Read block 176 len 24576 04/30 Write block 158 len 24576 data: 7 8 9 12 11 13 05/30 Read block 83 len 8192 06/30 Read block 95 len 28672 07/30 Read block 122 len 28672 08/30 Write block 410 len 12288 data: 9 8 8 09/30 Write block 0 len 32768 data: 3 3 6 9 11 12 14 13 10/30 Read block 240 len 8192 11/30 Read block 15 len 12288 12/30 Write block 413 len 16384 data: 7 7 8 7 13/30 Read block 572 len 20480 14/30 Write block 578 len 28672 data: 12 12 14 15 14 14 15 15/30 Write block 207 len 16384 data: 14 14 14 12 16/30 Read block 242 len 24576 17/30 Read block 148 len 28672 18/30 Read block 511 len 8192 19/30 Read block 159 len 20480 20/30 Write block 579 len 8192 data: 13 15 21/30 Read block 139 len 28672 22/30 Write block 591 len 32768 data: 4 6 5 4 4 5 5 2 23/30 Write block 137 len 28672 data: 12 13 15 13 10 8 10 24/30 Write block 100 len 4096 data: 12 25/30 Write block 174 len 36864 data: 11 11 14 14 10 12 15 10 10 26/30 Write block 219 len 12288 data: 9 10 8 27/30 Read block 207 len 40960 28/30 Write block 261 len 12288 data: 15 15 15 29/30 Read block 384 len 20480 30/30 Read block 221 len 28672 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Read 2 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 19 0 19 Done 30 11 30 71 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:17.30012484Z","hostname":"bmat-EVT22200007-0000ead9","pid":1688} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 208 208 208 T T T 1 020-039 120 120 120 218 218 218 F F F 2 040-059 122 122 122 221 221 221 F F F 3 060-079 123 123 123 222 222 222 T T T 4 080-099 118 118 118 211 211 211 F F F 5 100-119 121 121 121 220 220 220 T F T <--- 6 120-139 115 115 115 208 208 208 T F T <--- 7 140-159 119 119 119 213 213 213 T T T 8 160-179 123 123 123 222 222 222 T T T 9 180-199 123 123 123 222 222 222 T F T <--- 10 200-219 123 123 123 222 222 222 T F T <--- 11 220-239 123 123 123 222 222 222 T F T <--- 12 240-259 115 115 115 208 208 208 T T T 13 260-279 122 122 122 221 221 221 T F T <--- 14 280-299 123 123 123 222 222 222 F F F 15 300-319 118 118 118 212 212 212 F F F 16 320-339 122 122 122 221 221 221 F F F 17 340-359 122 122 122 221 221 221 F F F 18 360-379 117 117 117 210 210 210 F F F 19 380-399 120 120 120 214 214 214 F F F 20 400-419 112 112 112 206 206 206 T T T 21 420-439 118 118 118 212 212 212 F F F 22 440-459 119 119 119 213 213 213 F F F 23 460-479 123 123 123 222 222 222 F F F 24 480-499 120 120 120 214 214 214 F F F 25 500-519 120 120 120 219 219 219 F F F 26 520-539 120 120 120 219 219 219 F F F 27 540-559 120 120 120 216 216 216 F F F 28 560-579 120 120 120 219 219 219 T F T <--- 29 580-599 120 120 120 219 219 219 T F T <--- Max gen: 123, Max flush: 222 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 F F F 1 020-039 120 120 120 218 218 218 F F F 2 040-059 122 122 122 221 221 221 F F F 3 060-079 124 124 124 223 223 223 F F F 4 080-099 118 118 118 211 211 211 F F F 5 100-119 124 124 124 223 223 223 F F F 6 120-139 124 124 124 223 223 223 F F F 7 140-159 124 124 124 223 223 223 F F F 8 160-179 124 124 124 223 223 223 F F F 9 180-199 124 124 124 223 223 223 F F F 10 200-219 124 124 124 223 223 223 F F F 11 220-239 124 124 124 223 223 223 F F F 12 240-259 124 124 124 223 223 223 F F F 13 260-279 124 124 124 223 223 223 F F F 14 280-299 123 123 123 222 222 222 F F F 15 300-319 118 118 118 212 212 212 F F F 16 320-339 122 122 122 221 221 221 F F F 17 340-359 122 122 122 221 221 221 F F F 18 360-379 117 117 117 210 210 210 F F F 19 380-399 120 120 120 214 214 214 F F F 20 400-419 124 124 124 223 223 223 F F F 21 420-439 118 118 118 212 212 212 F F F 22 440-459 119 119 119 213 213 213 F F F 23 460-479 123 123 123 222 222 222 F F F 24 480-499 120 120 120 214 214 214 F F F 25 500-519 120 120 120 219 219 219 F F F 26 520-539 120 120 120 219 219 219 F F F 27 540-559 120 120 120 216 216 216 F F F 28 560-579 124 124 124 223 223 223 F F F 29 580-599 124 124 124 223 223 223 F F F Max gen: 124, Max flush: 223 Begin loop 63 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:21.959897964Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:21.985285778Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:21.985386378Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.009746828Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.010262489Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.010338419Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.010363709Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.010402416Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696} {"msg":"Crucible b5b79550-8f9b-47c7-b5f7-de0837f5bb13 has session id: 716eef1f-580a-43a8-b413-083ea9987ec2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.011291266Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"Upstairs opts: Upstairs UUID: b5b79550-8f9b-47c7-b5f7-de0837f5bb13, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.011352998Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"Crucible stats registered with UUID: b5b79550-8f9b-47c7-b5f7-de0837f5bb13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.011389129Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:22.011416704Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.012014588Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.012074416Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.012105097Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.012373488Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.01240446Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696} {"msg":"b5b79550-8f9b-47c7-b5f7-de0837f5bb13 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.012519978Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.012550669Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"client":"0","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.012580749Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"client":"1","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.012610478Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"client":"2","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.012639876Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.012670366Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.013294672Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"ds_connection connected from Ok(127.0.0.1:51809)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.013339491Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"ds_connection connected from Ok(127.0.0.1:37983)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.013367697Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"ds_connection connected from Ok(127.0.0.1:44140)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.01339973Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.0150108Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"client":"2","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.015196858Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"client":"1","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.015737579Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"client":"0","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.015824541Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016069476Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"[0]R flush_numbers[0..12]: [223, 218, 221, 223, 211, 223, 223, 223, 223, 223, 223, 223]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016458837Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"downstairs","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"[0]R generation[0..12]: [124, 120, 122, 124, 118, 124, 124, 124, 124, 124, 124, 124]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016503385Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"downstairs","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016543284Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"downstairs","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"[1]R flush_numbers[0..12]: [223, 218, 221, 223, 211, 223, 223, 223, 223, 223, 223, 223]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016581239Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"downstairs","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"[1]R generation[0..12]: [124, 120, 122, 124, 118, 124, 124, 124, 124, 124, 124, 124]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016618994Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"downstairs","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016658031Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"downstairs","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"[2]R flush_numbers[0..12]: [223, 218, 221, 223, 211, 223, 223, 223, 223, 223, 223, 223]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016695786Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"downstairs","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"[2]R generation[0..12]: [124, 120, 122, 124, 118, 124, 124, 124, 124, 124, 124, 124]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.0167334Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"downstairs","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016780163Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"downstairs","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"Max found gen is 125","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016812577Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"downstairs","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"Generation requested: 126 >= found:125","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016846104Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"downstairs","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016878247Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"downstairs","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016909719Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"":"downstairs","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016943136Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.016972774Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"client":"0","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-04T01:14:22.01700597Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"client":"1","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.017058955Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"client":"2","":"downstairs","session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.017092802Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.017124465Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"b5b79550-8f9b-47c7-b5f7-de0837f5bb13 is now active with session: 716eef1f-580a-43a8-b413-083ea9987ec2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.017159935Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.017189303Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696,"session_id":"716eef1f-580a-43a8-b413-083ea9987ec2"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.017220475Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696} {"msg":"Activated sub_volume b5b79550-8f9b-47c7-b5f7-de0837f5bb13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:22.017247278Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 496 len 20480 data: 9 9 8 8 6 02/30 Read block 277 len 32768 03/30 Read block 255 len 20480 04/30 Flush 05/30 Write block 62 len 12288 data: 6 7 10 06/30 Read block 12 len 32768 07/30 Write block 24 len 36864 data: 10 7 5 8 9 10 11 11 10 08/30 Write block 532 len 36864 data: 10 13 12 11 13 11 10 12 9 09/30 Read block 17 len 20480 10/30 Write block 192 len 36864 data: 13 16 15 11 13 11 12 10 12 11/30 Write block 92 len 8192 data: 9 12 12/30 Read block 288 len 12288 13/30 Read block 14 len 32768 14/30 Write block 318 len 8192 data: 8 9 15/30 Flush 16/30 Write block 534 len 4096 data: 13 17/30 Read block 278 len 20480 18/30 Write block 320 len 16384 data: 14 12 15 15 19/30 Read block 470 len 40960 20/30 Flush 21/30 Write block 456 len 12288 data: 11 14 16 22/30 Read block 560 len 36864 23/30 Write block 266 len 4096 data: 9 24/30 Write block 233 len 32768 data: 12 14 12 14 12 12 14 13 25/30 Flush 26/30 Write block 88 len 40960 data: 8 10 7 9 10 13 11 11 14 15 27/30 Read block 338 len 4096 28/30 Read block 274 len 20480 29/30 Write block 332 len 12288 data: 7 7 10 30/30 Read block 537 len 32768 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 3 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Write 9 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 1 0 27 Done 0 25 26 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:22.05464004Z","hostname":"bmat-EVT22200007-0000ead9","pid":1696} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 F F F 1 020-039 120 126 126 218 225 225 F F F <--- 2 040-059 122 122 122 221 221 221 F F F 3 060-079 124 126 126 223 225 225 F F F <--- 4 080-099 118 126 126 211 225 225 F T T <--- 5 100-119 124 124 124 223 223 223 F F F 6 120-139 124 124 124 223 223 223 F F F 7 140-159 124 124 124 223 223 223 F F F 8 160-179 124 124 124 223 223 223 F F F 9 180-199 124 126 126 223 225 225 F F F <--- 10 200-219 124 126 126 223 225 225 F F F <--- 11 220-239 124 126 126 223 227 227 F F F <--- 12 240-259 124 126 126 223 227 227 F F F <--- 13 260-279 124 126 126 223 227 227 F F F <--- 14 280-299 123 123 123 222 222 222 F F F 15 300-319 118 126 126 212 225 225 F F F <--- 16 320-339 122 126 126 221 226 226 F T T <--- 17 340-359 122 122 122 221 221 221 F F F 18 360-379 117 117 117 210 210 210 F F F 19 380-399 120 120 120 214 214 214 F F F 20 400-419 124 124 124 223 223 223 F F F 21 420-439 118 118 118 212 212 212 F F F 22 440-459 119 126 126 213 227 227 F F F <--- 23 460-479 123 123 123 222 222 222 F F F 24 480-499 126 126 126 224 224 224 F F F 25 500-519 126 126 126 224 224 224 F F F 26 520-539 120 126 126 219 226 226 F F F <--- 27 540-559 120 126 126 216 225 225 F F F <--- 28 560-579 124 124 124 223 223 223 F F F 29 580-599 124 124 124 223 223 223 F F F Max gen: 126, Max flush: 227 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 124 124 223 223 223 F F F 1 020-039 126 126 126 225 225 225 F F F 2 040-059 122 122 122 221 221 221 F F F 3 060-079 126 126 126 225 225 225 F F F 4 080-099 127 127 127 228 228 228 F F F 5 100-119 124 124 124 223 223 223 F F F 6 120-139 124 124 124 223 223 223 F F F 7 140-159 124 124 124 223 223 223 F F F 8 160-179 124 124 124 223 223 223 F F F 9 180-199 126 126 126 225 225 225 F F F 10 200-219 126 126 126 225 225 225 F F F 11 220-239 126 126 126 227 227 227 F F F 12 240-259 126 126 126 227 227 227 F F F 13 260-279 126 126 126 227 227 227 F F F 14 280-299 123 123 123 222 222 222 F F F 15 300-319 126 126 126 225 225 225 F F F 16 320-339 127 127 127 228 228 228 F F F 17 340-359 122 122 122 221 221 221 F F F 18 360-379 117 117 117 210 210 210 F F F 19 380-399 120 120 120 214 214 214 F F F 20 400-419 124 124 124 223 223 223 F F F 21 420-439 118 118 118 212 212 212 F F F 22 440-459 126 126 126 227 227 227 F F F 23 460-479 123 123 123 222 222 222 F F F 24 480-499 126 126 126 224 224 224 F F F 25 500-519 126 126 126 224 224 224 F F F 26 520-539 126 126 126 226 226 226 F F F 27 540-559 126 126 126 225 225 225 F F F 28 560-579 124 124 124 223 223 223 F F F 29 580-599 124 124 124 223 223 223 F F F Max gen: 127, Max flush: 228 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.692154266Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:26.7245169Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.724594663Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.755806248Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.756461275Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.756526965Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.756549349Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.756581984Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704} {"msg":"Crucible 86bda36e-a939-4b39-b5db-c456f5a47917 has session id: 3a885e92-72a3-4744-b5a8-4b4a77750bfa","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.757666431Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"Upstairs opts: Upstairs UUID: 86bda36e-a939-4b39-b5db-c456f5a47917, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.757810757Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"Crucible stats registered with UUID: 86bda36e-a939-4b39-b5db-c456f5a47917","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.757869343Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:26.758088687Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.758143776Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.758191Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.758219566Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.758449642Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.758484621Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704} {"msg":"86bda36e-a939-4b39-b5db-c456f5a47917 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.758603416Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.758632784Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"client":"0","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.758660169Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"client":"1","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.758687352Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"client":"2","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.758713765Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.758757832Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.758786118Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"ds_connection connected from Ok(127.0.0.1:49015)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.75923651Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"ds_connection connected from Ok(127.0.0.1:41117)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.759321909Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"ds_connection connected from Ok(127.0.0.1:47857)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.759360565Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.760993538Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"client":"2","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.761053798Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"client":"1","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.761259345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"client":"0","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.761813522Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.761898961Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"[0]R flush_numbers[0..12]: [223, 225, 221, 225, 228, 223, 223, 223, 223, 225, 225, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762268623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"downstairs","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"[0]R generation[0..12]: [124, 126, 122, 126, 127, 124, 124, 124, 124, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762334203Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"downstairs","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762366226Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"downstairs","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"[1]R flush_numbers[0..12]: [223, 225, 221, 225, 228, 223, 223, 223, 223, 225, 225, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762395023Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"downstairs","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"[1]R generation[0..12]: [124, 126, 122, 126, 127, 124, 124, 124, 124, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762423439Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"downstairs","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762452918Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"downstairs","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"[2]R flush_numbers[0..12]: [223, 225, 221, 225, 228, 223, 223, 223, 223, 225, 225, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762480883Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"downstairs","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [124, 126, 122, 126, 127, 124, 124, 124, 124, 126, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762520261Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"downstairs","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.76255027Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"downstairs","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762574318Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"downstairs","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762599187Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"downstairs","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762622584Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"downstairs","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.76264612Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"":"downstairs","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762670599Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762700528Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"client":"0","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762725938Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"client":"1","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762750888Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"client":"2","":"downstairs","session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762776178Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762799875Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"86bda36e-a939-4b39-b5db-c456f5a47917 is now active with session: 3a885e92-72a3-4744-b5a8-4b4a77750bfa","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762826247Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762848491Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704,"session_id":"3a885e92-72a3-4744-b5a8-4b4a77750bfa"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762872078Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704} {"msg":"Activated sub_volume 86bda36e-a939-4b39-b5db-c456f5a47917","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:26.762892328Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 509 len 28672 data: 9 11 8 8 8 9 8 02/30 Flush 03/30 Write block 17 len 24576 data: 6 6 9 10 11 12 04/30 Flush 05/30 Write block 92 len 36864 data: 11 14 12 12 15 16 14 13 13 06/30 Read block 380 len 8192 07/30 Read block 484 len 12288 08/30 Write block 463 len 40960 data: 12 11 10 12 11 8 9 11 10 9 09/30 Read block 554 len 32768 10/30 Flush 11/30 Write block 274 len 40960 data: 11 10 11 10 11 9 9 9 8 10 12/30 Read block 53 len 8192 13/30 Read block 36 len 28672 14/30 Write block 527 len 12288 data: 7 9 11 15/30 Flush 16/30 Flush 17/30 Write block 494 len 12288 data: 8 10 10 18/30 Write block 422 len 16384 data: 7 8 9 9 19/30 Flush 20/30 Write block 59 len 24576 data: 7 8 9 7 8 11 21/30 Write block 585 len 32768 data: 12 11 5 5 5 6 5 7 22/30 Flush 23/30 Write block 136 len 16384 data: 13 13 14 16 24/30 Read block 565 len 32768 25/30 Write block 93 len 16384 data: 15 13 13 16 26/30 Read block 298 len 40960 27/30 Write block 475 len 4096 data: 11 28/30 Read block 129 len 4096 29/30 Write block 179 len 16384 data: 13 16 11 11 30/30 Read block 532 len 28672 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Sent Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 6 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Read 2 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:26.801330299Z","hostname":"bmat-EVT22200007-0000ead9","pid":1704} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 124 128 128 223 230 230 F F F <--- 1 020-039 126 128 128 225 230 230 F F F <--- 2 040-059 122 128 128 221 235 235 F F F <--- 3 060-079 126 128 128 225 235 235 F F F <--- 4 080-099 127 128 128 228 231 231 F T T <--- 5 100-119 124 128 128 223 231 231 F F F <--- 6 120-139 124 124 124 223 223 223 F T T <--- 7 140-159 124 124 124 223 223 223 F F F 8 160-179 124 124 124 223 223 223 F T T <--- 9 180-199 126 126 126 225 225 225 F T T <--- 10 200-219 126 126 126 225 225 225 F F F 11 220-239 126 126 126 227 227 227 F F F 12 240-259 126 126 126 227 227 227 F F F 13 260-279 126 128 128 227 232 232 F F F <--- 14 280-299 123 128 128 222 232 232 F F F <--- 15 300-319 126 126 126 225 225 225 F F F 16 320-339 127 127 127 228 228 228 F F F 17 340-359 122 122 122 221 221 221 F F F 18 360-379 117 117 117 210 210 210 F F F 19 380-399 120 120 120 214 214 214 F F F 20 400-419 124 124 124 223 223 223 F F F 21 420-439 118 128 128 212 234 234 F F F <--- 22 440-459 126 126 126 227 227 227 F F F 23 460-479 123 128 128 222 231 231 F T T <--- 24 480-499 126 128 128 224 234 234 F F F <--- 25 500-519 126 128 128 224 229 229 F F F <--- 26 520-539 126 128 128 226 232 232 F F F <--- 27 540-559 126 126 126 225 225 225 F F F 28 560-579 124 124 124 223 223 223 F F F 29 580-599 124 128 128 223 235 235 F F F <--- Max gen: 128, Max flush: 235 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 230 230 230 F F F 1 020-039 128 128 128 230 230 230 F F F 2 040-059 128 128 128 235 235 235 F F F 3 060-079 128 128 128 235 235 235 F F F 4 080-099 129 129 129 236 236 236 F F F 5 100-119 128 128 128 231 231 231 F F F 6 120-139 129 129 129 236 236 236 F F F 7 140-159 124 124 124 223 223 223 F F F 8 160-179 129 129 129 236 236 236 F F F 9 180-199 129 129 129 236 236 236 F F F 10 200-219 126 126 126 225 225 225 F F F 11 220-239 126 126 126 227 227 227 F F F 12 240-259 126 126 126 227 227 227 F F F 13 260-279 128 128 128 232 232 232 F F F 14 280-299 128 128 128 232 232 232 F F F 15 300-319 126 126 126 225 225 225 F F F 16 320-339 127 127 127 228 228 228 F F F 17 340-359 122 122 122 221 221 221 F F F 18 360-379 117 117 117 210 210 210 F F F 19 380-399 120 120 120 214 214 214 F F F 20 400-419 124 124 124 223 223 223 F F F 21 420-439 128 128 128 234 234 234 F F F 22 440-459 126 126 126 227 227 227 F F F 23 460-479 129 129 129 236 236 236 F F F 24 480-499 128 128 128 234 234 234 F F F 25 500-519 128 128 128 229 229 229 F F F 26 520-539 128 128 128 232 232 232 F F F 27 540-559 126 126 126 225 225 225 F F F 28 560-579 124 124 124 223 223 223 F F F 29 580-599 128 128 128 235 235 235 F F F Max gen: 129, Max flush: 236 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.489152848Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.52274211Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.523542194Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.523616892Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.523642262Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.523678303Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712} {"msg":"Crucible e16f539e-6873-441b-96d1-bf7ec6dc0578 has session id: a9064376-a9f6-45d3-aba0-5149674891e2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.524980181Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"Upstairs opts: Upstairs UUID: e16f539e-6873-441b-96d1-bf7ec6dc0578, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.525145428Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"Crucible stats registered with UUID: e16f539e-6873-441b-96d1-bf7ec6dc0578","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.525201228Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:31.525244975Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.52578786Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.525874732Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.525919821Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.526122993Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.526172702Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712} {"msg":"e16f539e-6873-441b-96d1-bf7ec6dc0578 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.526293862Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.526336526Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"client":"0","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.526400282Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"client":"1","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.526440933Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"client":"2","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.526476964Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.52651543Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.52655572Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:41659)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.526665868Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:48374)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.52672216Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:45634)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.52676282Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.52915681Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"client":"0","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.529223602Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"client":"2","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.530374381Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.530582603Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.531307278Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"client":"1","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"[0]R flush_numbers[0..12]: [230, 230, 235, 235, 236, 231, 236, 223, 236, 236, 225, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.532439088Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"downstairs","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"[0]R generation[0..12]: [128, 128, 128, 128, 129, 128, 129, 124, 129, 129, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.532490821Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"downstairs","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.532534076Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"downstairs","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"[1]R flush_numbers[0..12]: [230, 230, 235, 235, 236, 231, 236, 223, 236, 236, 225, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.532568094Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"downstairs","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"[1]R generation[0..12]: [128, 128, 128, 128, 129, 128, 129, 124, 129, 129, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.53260163Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"downstairs","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.53263664Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"downstairs","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"[2]R flush_numbers[0..12]: [230, 230, 235, 235, 236, 231, 236, 223, 236, 236, 225, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.532669996Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"downstairs","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"[2]R generation[0..12]: [128, 128, 128, 128, 129, 128, 129, 124, 129, 129, 126, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.532711668Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"downstairs","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.532747038Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"downstairs","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.532780184Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"downstairs","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.53282874Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"downstairs","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.532884691Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"downstairs","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.532938838Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"":"downstairs","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.532982905Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.533017173Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"client":"0","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.533052242Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"client":"1","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.533090528Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"client":"2","":"downstairs","session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.533122061Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.533150367Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"e16f539e-6873-441b-96d1-bf7ec6dc0578 is now active with session: a9064376-a9f6-45d3-aba0-5149674891e2","v":0,"name":Wait for a query_work_queue command to finish before sending IO "crucible","level":30,"time":"2026-03-04T01:14:31.533196708Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.533224433Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712,"session_id":"a9064376-a9f6-45d3-aba0-5149674891e2"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.533252399Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712} {"msg":"Activated sub_volume e16f539e-6873-441b-96d1-bf7ec6dc0578","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:31.533276506Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 262 len 24576 data: 16 16 11 8 10 7 02/30 Read block 155 len 20480 03/30 Read block 174 len 40960 04/30 Read block 479 len 24576 05/30 Write block 422 len 16384 data: 8 9 10 10 06/30 Read block 536 len 8192 07/30 Read block 59 len 4096 08/30 Write block 467 len 36864 data: 12 9 10 12 11 10 10 10 12 09/30 Flush 10/30 Write block 467 len 28672 data: 13 10 11 13 12 11 11 11/30 Read block 21 len 16384 12/30 Read block 76 len 36864 13/30 Read block 33 len 4096 14/30 Write block 260 len 36864 data: 15 16 17 17 12 9 11 8 10 15/30 Read block 181 len 4096 16/30 Write block 36 len 8192 data: 9 8 17/30 Write block 209 len 20480 data: 15 13 10 11 8 18/30 Read block 580 len 40960 19/30 Read block 22 len 20480 20/30 Write block 540 len 24576 data: 10 10 12 11 9 10 21/30 Read block 530 len 40960 22/30 Read block 391 len 40960 23/30 Write block 595 len 20480 data: 5 6 6 3 3 24/30 Read block 229 len 4096 25/30 Read block 104 len 28672 26/30 Read block 149 len 28672 27/30 Write block 538 len 20480 data: 11 13 11 11 13 28/30 Write block 279 len 36864 data: 10 10 10 9 11 14 11 11 11 29/30 Write block 357 len 36864 data: 12 16 15 14 13 14 11 11 12 30/30 Read block 6 len 8192 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Read 6 Done Sent Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) None Some(JobId(1008)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:31.575945265Z","hostname":"bmat-EVT22200007-0000ead9","pid":1712} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 230 230 230 F F F 1 020-039 128 128 128 230 230 230 T F T <--- 2 040-059 128 128 128 235 235 235 F F F 3 060-079 128 128 128 235 235 235 F F F 4 080-099 129 129 129 236 236 236 F F F 5 100-119 128 128 128 231 231 231 F F F 6 120-139 129 129 129 236 236 236 F F F 7 140-159 124 124 124 223 223 223 F F F 8 160-179 129 129 129 236 236 236 F F F 9 180-199 129 129 129 236 236 236 F F F 10 200-219 126 126 126 225 225 225 T F T <--- 11 220-239 126 126 126 227 227 227 F F F 12 240-259 126 126 126 227 227 227 F F F 13 260-279 130 128 130 237 232 237 T T T <--- 14 280-299 128 128 128 232 232 232 T F T <--- 15 300-319 126 126 126 225 225 225 F F F 16 320-339 127 127 127 228 228 228 F F F 17 340-359 122 122 122 221 221 221 T F T <--- 18 360-379 117 117 117 210 210 210 T F T <--- 19 380-399 120 120 120 214 214 214 F F F 20 400-419 124 124 124 223 223 223 F F F 21 420-439 130 128 130 237 234 237 F F F <--- 22 440-459 126 126 126 227 227 227 F F F 23 460-479 130 129 130 237 236 237 T F T <--- 24 480-499 128 128 128 234 234 234 F F F 25 500-519 128 128 128 229 229 229 F F F 26 520-539 128 128 128 232 232 232 T F T <--- 27 540-559 126 126 126 225 225 225 T F T <--- 28 560-579 124 124 124 223 223 223 F F F 29 580-599 128 128 128 235 235 235 T F T <--- Max gen: 130, Max flush: 237 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 230 230 230 F F F 1 020-039 131 131 131 238 238 238 F F F 2 040-059 128 128 128 235 235 235 F F F 3 060-079 128 128 128 235 235 235 F F F 4 080-099 129 129 129 236 236 236 F F F 5 100-119 128 128 128 231 231 231 F F F 6 120-139 129 129 129 236 236 236 F F F 7 140-159 124 124 124 223 223 223 F F F 8 160-179 129 129 129 236 236 236 F F F 9 180-199 129 129 129 236 236 236 F F F 10 200-219 131 131 131 238 238 238 F F F 11 220-239 126 126 126 227 227 227 F F F 12 240-259 126 126 126 227 227 227 F F F 13 260-279 131 131 131 238 238 238 F F F 14 280-299 131 131 131 238 238 238 F F F 15 300-319 126 126 126 225 225 225 F F F 16 320-339 127 127 127 228 228 228 F F F 17 340-359 131 131 131 238 238 238 F F F 18 360-379 131 131 131 238 238 238 F F F 19 380-399 120 120 120 214 214 214 F F F 20 400-419 124 124 124 223 223 223 F F F 21 420-439 130 130 130 237 237 237 F F F 22 440-459 126 126 126 227 227 227 F F F 23 460-479 131 131 131 238 238 238 F F F 24 480-499 128 128 128 234 234 234 F F F 25 500-519 128 128 128 229 229 229 F F F 26 520-539 131 131 131 238 238 238 F F F 27 540-559 131 131 131 238 238 238 F F F 28 560-579 124 124 124 223 223 223 F F F 29 580-599 131 131 131 238 238 238 F F F Max gen: 131, Max flush: 238 Begin loop 66 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.232955307Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:36.259561434Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.259650781Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.284385021Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.28494006Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.28502584Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.285052783Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.285093524Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720} {"msg":"Crucible c3dfd6b5-3b5d-458d-acad-695b3af51476 has session id: a2313310-a24c-47d0-bebb-769e02d9a002","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.286257819Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"Upstairs opts: Upstairs UUID: c3dfd6b5-3b5d-458d-acad-695b3af51476, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.286345122Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"Crucible stats registered with UUID: c3dfd6b5-3b5d-458d-acad-695b3af51476","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.286385321Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:36.286417445Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.286942504Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.287104475Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.28715757Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.287196988Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.287228431Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720} {"msg":"c3dfd6b5-3b5d-458d-acad-695b3af51476 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.28726339Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.287301676Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"client":"0","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.287341945Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"client":"1","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.287388097Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"client":"2","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.287433958Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.28748085Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.287883848Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"ds_connection connected from Ok(127.0.0.1:44770)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.287981852Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"ds_connection connected from Ok(127.0.0.1:35582)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.288017854Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"ds_connection connected from Ok(127.0.0.1:40191)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.288045659Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.290447243Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"client":"1","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.29055661Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"client":"0","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.290618903Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"client":"2","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.291413667Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.291562712Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"[0]R flush_numbers[0..12]: [230, 238, 235, 235, 236, 231, 236, 223, 236, 236, 238, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.291848037Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"downstairs","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"[0]R generation[0..12]: [128, 131, 128, 128, 129, 128, 129, 124, 129, 129, 131, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.291960519Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"downstairs","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292017372Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"downstairs","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"[1]R flush_numbers[0..12]: [230, 238, 235, 235, 236, 231, 236, 223, 236, 236, 238, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292073704Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"downstairs","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"[1]R generation[0..12]: [128, 131, 128, 128, 129, 128, 129, 124, 129, 129, 131, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.29213265Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"downstairs","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292189623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"downstairs","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"[2]R flush_numbers[0..12]: [230, 238, 235, 235, 236, 231, 236, 223, 236, 236, 238, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292251876Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"downstairs","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [128, 131, 128, 128, 129, 128, 129, 124, 129, 129, 131, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292316314Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"downstairs","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292372555Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"downstairs","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292422825Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"downstairs","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292471381Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"downstairs","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292519196Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"downstairs","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292577852Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"":"downstairs","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292628622Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292676617Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"bmat-EVT22200007-0000ead9","pid":1720,"client":"0","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292732839Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"client":"1","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292781555Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"client":"2","":"downstairs","session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292835261Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292881533Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"c3dfd6b5-3b5d-458d-acad-695b3af51476 is now active with session: a2313310-a24c-47d0-bebb-769e02d9a002","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.292933075Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.29298105Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720,"session_id":"a2313310-a24c-47d0-bebb-769e02d9a002"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.29302676Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720} {"msg":"Activated sub_volume c3dfd6b5-3b5d-458d-acad-695b3af51476","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:36.293066189Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 241 len 36864 data: 12 10 9 8 9 9 9 9 11 02/30 Read block 472 len 40960 03/30 Read block 480 len 36864 04/30 Write block 575 len 24576 data: 11 9 11 13 14 16 05/30 Read block 370 len 12288 06/30 Read block 539 len 12288 07/30 Read block 63 len 24576 08/30 Read block 428 len 24576 09/30 Read block 285 len 16384 10/30 Write block 436 len 32768 data: 15 18 17 13 12 10 10 7 11/30 Read block 294 len 32768 12/30 Write block 146 len 24576 data: 11 13 12 8 7 9 13/30 Write block 18 len 12288 data: 7 10 11 14/30 Read block 298 len 32768 15/30 Read block 310 len 20480 16/30 Read block 92 len 40960 17/30 Read block 12 len 4096 18/30 Read block 275 len 16384 19/30 Write block 546 len 4096 data: 8 20/30 Write block 82 len 36864 data: 8 6 8 8 9 9 9 11 8 21/30 Read block 294 len 4096 22/30 Read block 38 len 24576 23/30 Write block 63 len 28672 data: 9 12 8 6 6 10 11 24/30 Read block 357 len 16384 25/30 Flush 26/30 Write block 496 len 24576 data: 11 10 9 9 7 6 27/30 Write block 242 len 8192 data: 11 10 28/30 Write block 204 len 4096 data: 7 29/30 Read block 314 len 20480 30/30 Read block 432 len 12288 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Read 3 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Read 6 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Read 5 Sent Done Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 1 0 27 Done 4 29 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:36.329475845Z","hostname":"bmat-EVT22200007-0000ead9","pid":1720} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 132 132 230 239 239 F F F <--- 1 020-039 131 132 132 238 239 239 F F F <--- 2 040-059 128 128 128 235 235 235 F F F 3 060-079 128 132 132 235 239 239 F F F <--- 4 080-099 129 132 132 236 239 239 F F F <--- 5 100-119 128 128 128 231 231 231 F F F 6 120-139 129 129 129 236 236 236 F F F 7 140-159 124 132 132 223 239 239 F F F <--- 8 160-179 129 129 129 236 236 236 F F F 9 180-199 129 129 129 236 236 236 F F F 10 200-219 131 131 131 238 238 238 F T T <--- 11 220-239 126 126 126 227 227 227 F F F 12 240-259 126 132 132 227 239 239 T T T <--- 13 260-279 131 131 131 238 238 238 F F F 14 280-299 131 131 131 238 238 238 F F F 15 300-319 126 126 126 225 225 225 F F F 16 320-339 127 127 127 228 228 228 F F F 17 340-359 131 131 131 238 238 238 F F F 18 360-379 131 131 131 238 238 238 F F F 19 380-399 120 120 120 214 214 214 F F F 20 400-419 124 124 124 223 223 223 F F F 21 420-439 130 132 132 237 239 239 F F F <--- 22 440-459 126 132 132 227 239 239 F F F <--- 23 460-479 131 131 131 238 238 238 F F F 24 480-499 128 128 128 234 234 234 F T T <--- 25 500-519 128 128 128 229 229 229 F T T <--- 26 520-539 131 131 131 238 238 238 F F F 27 540-559 131 132 132 238 239 239 F F F <--- 28 560-579 124 132 132 223 239 239 T F F <--- 29 580-599 131 132 132 238 239 239 T F F <--- Max gen: 132, Max flush: 239 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 132 132 132 239 239 239 F F F 2 040-059 128 128 128 235 235 235 F F F 3 060-079 132 132 132 239 239 239 F F F 4 080-099 132 132 132 239 239 239 F F F 5 100-119 128 128 128 231 231 231 F F F 6 120-139 129 129 129 236 236 236 F F F 7 140-159 132 132 132 239 239 239 F F F 8 160-179 129 129 129 236 236 236 F F F 9 180-199 129 129 129 236 236 236 F F F 10 200-219 133 133 133 240 240 240 F F F 11 220-239 126 126 126 227 227 227 F F F 12 240-259 133 133 133 240 240 240 F F F 13 260-279 131 131 131 238 238 238 F F F 14 280-299 131 131 131 238 238 238 F F F 15 300-319 126 126 126 225 225 225 F F F 16 320-339 127 127 127 228 228 228 F F F 17 340-359 131 131 131 238 238 238 F F F 18 360-379 131 131 131 238 238 238 F F F 19 380-399 120 120 120 214 214 214 F F F 20 400-419 124 124 124 223 223 223 F F F 21 420-439 132 132 132 239 239 239 F F F 22 440-459 132 132 132 239 239 239 F F F 23 460-479 131 131 131 238 238 238 F F F 24 480-499 133 133 133 240 240 240 F F F 25 500-519 133 133 133 240 240 240 F F F 26 520-539 131 131 131 238 238 238 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 132 132 132 239 239 239 F F F 29 580-599 132 132 132 239 239 239 F F F Max gen: 133, Max flush: 240 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.040174434Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.067968713Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.068624852Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.068720301Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.068756493Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.068810209Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728} {"msg":"Crucible cd33cb44-01f1-450c-b386-5f95b1fa189c has session id: adfca7db-2500-40b2-87df-e51a4eea52dd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.069877212Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"Upstairs opts: Upstairs UUID: cd33cb44-01f1-450c-b386-5f95b1fa189c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.069963292Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"Crucible stats registered with UUID: cd33cb44-01f1-450c-b386-5f95b1fa189c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.070016247Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:41.070059082Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.070507409Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.070587097Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.070637407Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.070872623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.070923163Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728} {"msg":"cd33cb44-01f1-450c-b386-5f95b1fa189c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.071175131Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.07122514Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"client":"0","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.07126533Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"client":"1","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.071306231Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"client":"2","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.071364977Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.071413844Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.071459094Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:48526)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.071798696Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:65240)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.071898834Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:62939)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.071952671Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.073828505Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"client":"0","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.073879345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"client":"2","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.07485137Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"client":"1","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.074962801Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.075014463Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"[0]R flush_numbers[0..12]: [239, 239, 235, 239, 239, 231, 236, 239, 236, 236, 240, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076141685Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"downstairs","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"[0]R generation[0..12]: [132, 132, 128, 132, 132, 128, 129, 132, 129, 129, 133, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076213447Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"downstairs","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076251352Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"downstairs","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"[1]R flush_numbers[0..12]: [239, 239, 235, 239, 239, 231, 236, 239, 236, 236, 240, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076283285Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"downstairs","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"[1]R generation[0..12]: [132, 132, 128, 132, 132, 128, 129, 132, 129, 129, 133, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076314026Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"downstairs","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076357392Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"downstairs","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"[2]R flush_numbers[0..12]: [239, 239, 235, 239, 239, 231, 236, 239, 236, 236, 240, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076388363Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"downstairs","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"[2]R generation[0..12]: [132, 132, 128, 132, 132, 128, 129, 132, 129, 129, 133, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076420196Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"downstairs","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076451939Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"downstairs","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076478201Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"downstairs","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076505294Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"downstairs","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076530825Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"downstairs","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076556486Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076598549Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076623438Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"client":"0","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076650853Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"client":"1","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076677876Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"client":"2","":"downstairs","session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076705531Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076731112Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"cd33cb44-01f1-450c-b386-5f95b1fa189c is now active with session: adfca7db-2500-40b2-87df-e51a4eea52dd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076759929Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076796401Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728,"session_id":"adfca7db-2500-40b2-87df-e51a4eea52dd"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076822452Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728} {"msg":"Activated sub_volume cd33cb44-01f1-450c-b386-5f95b1fa189c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:41.076844767Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 376 len 24576 data: 7 8 9 10 9 9 02/30 Write block 509 len 16384 data: 10 12 9 9 03/30 Read block 100 len 28672 04/30 Read block 310 len 40960 05/30 Write block 312 len 8192 data: 8 6 06/30 Read block 339 len 20480 07/30 Read block 169 len 36864 08/30 Read block 494 len 40960 09/30 Read block 528 len 20480 10/30 Read block 557 len 16384 11/30 Write block 594 len 24576 data: 5 6 7 7 4 4 12/30 Write block 350 len 4096 data: 7 13/30 Read block 22 len 24576 14/30 Read block 61 len 36864 15/30 Write block 277 len 28672 data: 11 12 11 11 11 10 12 16/30 Write block 270 len 32768 data: 12 12 12 11 12 11 12 12 17/30 Write block 521 len 20480 data: 6 9 8 7 8 18/30 Write block 371 len 16384 data: 10 10 10 11 19/30 Read block 482 len 32768 20/30 Read block 422 len 28672 21/30 Write block 442 len 32768 data: 11 8 9 10 10 8 10 11 22/30 Read block 40 len 24576 23/30 Write block 115 len 12288 data: 12 13 15 24/30 Read block 342 len 24576 25/30 Write block 451 len 4096 data: 7 26/30 Read block 512 len 4096 27/30 Write block 578 len 4096 data: 14 28/30 Write block 110 len 16384 data: 9 9 9 15 29/30 Write block 144 len 16384 data: 10 9 12 14 30/30 Read block 551 len 32768 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Sent Done false 1001 Acked Write 4 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:41.109119436Z","hostname":"bmat-EVT22200007-0000ead9","pid":1728} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 132 132 132 239 239 239 F F F 2 040-059 128 128 128 235 235 235 F F F 3 060-079 132 132 132 239 239 239 F F F 4 080-099 132 132 132 239 239 239 F F F 5 100-119 128 128 128 231 231 231 T F T <--- 6 120-139 129 129 129 236 236 236 F F F 7 140-159 132 132 132 239 239 239 T F T <--- 8 160-179 129 129 129 236 236 236 F F F 9 180-199 129 129 129 236 236 236 F F F 10 200-219 133 133 133 240 240 240 F F F 11 220-239 126 126 126 227 227 227 F F F 12 240-259 133 133 133 240 240 240 F F F 13 260-279 131 131 131 238 238 238 T F T <--- 14 280-299 131 131 131 238 238 238 T F T <--- 15 300-319 126 126 126 225 225 225 T F T <--- 16 320-339 127 127 127 228 228 228 F F F 17 340-359 131 131 131 238 238 238 T F T <--- 18 360-379 131 131 131 238 238 238 T F T <--- 19 380-399 120 120 120 214 214 214 T F T <--- 20 400-419 124 124 124 223 223 223 F F F 21 420-439 132 132 132 239 239 239 F F F 22 440-459 132 132 132 239 239 239 T F T <--- 23 460-479 131 131 131 238 238 238 F F F 24 480-499 133 133 133 240 240 240 F F F 25 500-519 133 133 133 240 240 240 T F T <--- 26 520-539 131 131 131 238 238 238 T F T <--- 27 540-559 132 132 132 239 239 239 F F F 28 560-579 132 132 132 239 239 239 T F T <--- 29 580-599 132 132 132 239 239 239 T F T <--- Max gen: 133, Max flush: 240 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 132 132 132 239 239 239 F F F 2 040-059 128 128 128 235 235 235 F F F 3 060-079 132 132 132 239 239 239 F F F 4 080-099 132 132 132 239 239 239 F F F 5 100-119 134 134 134 241 241 241 F F F 6 120-139 129 129 129 236 236 236 F F F 7 140-159 134 134 134 241 241 241 F F F 8 160-179 129 129 129 236 236 236 F F F 9 180-199 129 129 129 236 236 236 F F F 10 200-219 133 133 133 240 240 240 F F F 11 220-239 126 126 126 227 227 227 F F F 12 240-259 133 133 133 240 240 240 F F F 13 260-279 134 134 134 241 241 241 F F F 14 280-299 134 134 134 241 241 241 F F F 15 300-319 134 134 134 241 241 241 F F F 16 320-339 127 127 127 228 228 228 F F F 17 340-359 134 134 134 241 241 241 F F F 18 360-379 134 134 134 241 241 241 F F F 19 380-399 134 134 134 241 241 241 F F F 20 400-419 124 124 124 223 223 223 F F F 21 420-439 132 132 132 239 239 239 F F F 22 440-459 134 134 134 241 241 241 F F F 23 460-479 131 131 131 238 238 238 F F F 24 480-499 133 133 133 240 240 240 F F F 25 500-519 134 134 134 241 241 241 F F F 26 520-539 134 134 134 241 241 241 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 134 134 134 241 241 241 F F F 29 580-599 134 134 134 241 241 241 F F F Max gen: 134, Max flush: 241 Begin loop 68 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.773126312Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.80701151Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.807730162Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.807876502Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.807907534Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.807951421Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736} {"msg":"Crucible 75a5c78a-ca11-4cc8-bc5e-ba760bd4e718 has session id: 0843c0b2-f253-43f8-be3d-be92cbb8d0e9","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.808877053Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"Upstairs opts: Upstairs UUID: 75a5c78a-ca11-4cc8-bc5e-ba760bd4e718, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.808988384Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"Crucible stats registered with UUID: 75a5c78a-ca11-4cc8-bc5e-ba760bd4e718","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.809041338Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:45.809099043Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.809477251Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.809566819Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.809635124Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.809800752Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.80986052Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736} {"msg":"75a5c78a-ca11-4cc8-bc5e-ba760bd4e718 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.80990074Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.809949015Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"client":"0","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.809997742Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"client":"1","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.81004717Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"client":"2","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.810105345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61221)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.810158811Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.810213649Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.810266434Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:48006)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.81032015Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:38455)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.810585666Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.812994375Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"client":"1","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.813077149Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"client":"0","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.813654332Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.813795952Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.814662217Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"client":"2","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"[0]R flush_numbers[0..12]: [239, 239, 235, 239, 239, 241, 236, 241, 236, 236, 240, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.815597699Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"downstairs","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"[0]R generation[0..12]: [132, 132, 128, 132, 132, 134, 129, 134, 129, 129, 133, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.815667597Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"downstairs","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.815713959Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"downstairs","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"[1]R flush_numbers[0..12]: [239, 239, 235, 239, 239, 241, 236, 241, 236, 236, 240, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.815757225Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"downstairs","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"[1]R generation[0..12]: [132, 132, 128, 132, 132, 134, 129, 134, 129, 129, 133, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.815808176Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"downstairs","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.815852975Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"downstairs","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"[2]R flush_numbers[0..12]: [239, 239, 235, 239, 239, 241, 236, 241, 236, 236, 240, 227]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.815895729Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"downstairs","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"[2]R generation[0..12]: [132, 132, 128, 132, 132, 134, 129, 134, 129, 129, 133, 126]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.815938183Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"downstairs","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.815982501Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"downstairs","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"Max found gen is 135","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.816054594Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"downstairs","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"Generation requested: 136 >= found:135","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.816106597Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"downstairs","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:14:45.816142989Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"downstairs","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.816206715Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"":"downstairs","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.816247415Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.816281533Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"client":"0","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.816319268Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"client":"1","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.816356692Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"client":"2","":"downstairs","session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.816394917Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.816430388Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"75a5c78a-ca11-4cc8-bc5e-ba760bd4e718 is now active with session: 0843c0b2-f253-43f8-be3d-be92cbb8d0e9","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.816470567Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.816504194Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736,"session_id":"0843c0b2-f253-43f8-be3d-be92cbb8d0e9"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.816539794Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736} {"msg":"Activated sub_volume 75a5c78a-ca11-4cc8-bc5e-ba760bd4e718","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:45.816570144Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 589 len 8192 data: 6 7 02/30 Read block 543 len 36864 03/30 Read block 147 len 24576 04/30 Read block 389 len 32768 05/30 Read block 390 len 4096 06/30 Write block 305 len 16384 data: 13 13 14 14 07/30 Read block 127 len 20480 08/30 Read block 466 len 16384 09/30 Write block 413 len 16384 data: 8 8 9 8 10/30 Read block 256 len 32768 11/30 Write block 328 len 16384 data: 7 7 8 7 12/30 Flush 13/30 Write block 238 len 12288 data: 13 15 14 14/30 Read block 162 len 20480 15/30 Read block 114 len 4096 16/30 Write block 65 len 36864 data: 9 7 7 11 12 12 10 11 11 17/30 Write block 268 len 32768 data: 11 11 13 13 13 12 13 12 18/30 Read block 518 len 36864 19/30 Read block 336 len 40960 20/30 Write block 446 len 36864 data: 11 9 11 12 9 8 6 6 10 21/30 Read block 97 len 4096 22/30 Flush 23/30 Write block 426 len 12288 data: 11 10 10 24/30 Write block 485 len 32768 data: 7 7 8 6 6 7 6 6 25/30 Write block 96 len 8192 data: 17 17 26/30 Write block 220 len 36864 data: 11 9 7 6 7 7 8 7 8 27/30 Read block 15 len 36864 28/30 Read block 422 len 24576 29/30 Write block 111 len 40960 data: 10 10 16 14 13 14 16 14 12 13 30/30 Read block 30 len 24576 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Read 1 Done Done Sent false 1005 Acked Write 4 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Write 10 Done Sent Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 26 28 Done 30 28 4 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:45.858411304Z","hostname":"bmat-EVT22200007-0000ead9","pid":1736} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 132 132 132 239 239 239 F F F 2 040-059 128 128 128 235 235 235 F F F 3 060-079 136 136 132 243 243 239 F F F <--- 4 080-099 132 132 132 239 239 239 T T F <--- 5 100-119 134 134 134 241 241 241 T T F <--- 6 120-139 129 129 129 236 236 236 T T F <--- 7 140-159 134 134 134 241 241 241 F F F 8 160-179 129 129 129 236 236 236 F F F 9 180-199 129 129 129 236 236 236 F F F 10 200-219 133 133 133 240 240 240 F F F 11 220-239 136 136 126 243 243 227 T T F <--- 12 240-259 136 136 133 243 243 240 F F F <--- 13 260-279 136 136 134 243 243 241 F F F <--- 14 280-299 134 134 134 241 241 241 F F F 15 300-319 136 136 134 242 242 241 F F F <--- 16 320-339 136 136 127 242 242 228 F F F <--- 17 340-359 134 134 134 241 241 241 F F F 18 360-379 134 134 134 241 241 241 F F F 19 380-399 134 134 134 241 241 241 F F F 20 400-419 136 136 124 242 242 223 F F F <--- 21 420-439 132 132 132 239 239 239 T T F <--- 22 440-459 136 136 134 243 243 241 F F F <--- 23 460-479 131 131 131 238 238 238 F F F 24 480-499 133 133 133 240 240 240 T T F <--- 25 500-519 134 134 134 241 241 241 F F F 26 520-539 134 134 134 241 241 241 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 134 134 134 241 241 241 F F F 29 580-599 136 136 134 242 242 241 F F T <--- Max gen: 136, Max flush: 243 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 239 239 239 F F F 1 020-039 132 132 132 239 239 239 F F F 2 040-059 128 128 128 235 235 235 F F F 3 060-079 136 136 136 243 243 243 F F F 4 080-099 137 137 137 244 244 244 F F F 5 100-119 137 137 137 244 244 244 F F F 6 120-139 137 137 137 244 244 244 F F F 7 140-159 134 134 134 241 241 241 F F F 8 160-179 129 129 129 236 236 236 F F F 9 180-199 129 129 129 236 236 236 F F F 10 200-219 133 133 133 240 240 240 F F F 11 220-239 137 137 137 244 244 244 F F F 12 240-259 136 136 136 243 243 243 F F F 13 260-279 136 136 136 243 243 243 F F F 14 280-299 134 134 134 241 241 241 F F F 15 300-319 136 136 136 242 242 242 F F F 16 320-339 136 136 136 242 242 242 F F F 17 340-359 134 134 134 241 241 241 F F F 18 360-379 134 134 134 241 241 241 F F F 19 380-399 134 134 134 241 241 241 F F F 20 400-419 136 136 136 242 242 242 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 136 136 136 243 243 243 F F F 23 460-479 131 131 131 238 238 238 F F F 24 480-499 137 137 137 244 244 244 F F F 25 500-519 134 134 134 241 241 241 F F F 26 520-539 134 134 134 241 241 241 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 134 134 134 241 241 241 F F F 29 580-599 136 136 136 242 242 242 F F F Max gen: 137, Max flush: 244 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.596384253Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.623559477Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.624366214Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.624473617Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.624499388Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.624542753Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744} {"msg":"Crucible d82e9f72-da2d-4e3f-9d48-54779ea98aad has session id: 5095ca54-79d3-4a03-b6f7-01eb8918cedd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.625442956Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"Upstairs opts: Upstairs UUID: d82e9f72-da2d-4e3f-9d48-54779ea98aad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.625503526Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"Crucible stats registered with UUID: d82e9f72-da2d-4e3f-9d48-54779ea98aad","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.625533154Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:50.625558675Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.625963637Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.626017092Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.62604597Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.626496682Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.626540829Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744} {"msg":"d82e9f72-da2d-4e3f-9d48-54779ea98aad active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.626662761Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.626691097Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"client":"0","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.626717599Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"client":"1","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.626743962Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"client":"2","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.626848819Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.626881173Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.626913267Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"ds_connection connected from Ok(127.0.0.1:46261)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.62746523Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"ds_connection connected from Ok(127.0.0.1:44433)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.627538545Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"ds_connection connected from Ok(127.0.0.1:36231)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.627569386Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.629266847Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"client":"1","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.62941019Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"client":"2","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.629456121Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"client":"0","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630224593Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630437254Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"[0]R flush_numbers[0..12]: [239, 239, 235, 243, 244, 244, 244, 241, 236, 236, 240, 244]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630567732Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"downstairs","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"[0]R generation[0..12]: [132, 132, 128, 136, 137, 137, 137, 134, 129, 129, 133, 137]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630605948Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"downstairs","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630638933Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"downstairs","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"[1]R flush_numbers[0..12]: [239, 239, 235, 243, 244, 244, 244, 241, 236, 236, 240, 244]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630669804Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"downstairs","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"[1]R generation[0..12]: [132, 132, 128, 136, 137, 137, 137, 134, 129, 129, 133, 137]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630700104Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"downstairs","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630741356Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"downstairs","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"[2]R flush_numbers[0..12]: [239, 239, 235, 243, 244, 244, 244, 241, 236, 236, 240, 244]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630772357Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"downstairs","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"[2]R generation[0..12]: [132, 132, 128, 136, 137, 137, 137, 134, 129, 129, 133, 137]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630802637Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"downstairs","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630834981Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"downstairs","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630861223Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"downstairs","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630888397Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"downstairs","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630913887Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"downstairs","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630939348Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"":"downstairs","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.630966221Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.631015258Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"client":"0","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.631045117Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"client":"1","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.631072031Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"client":"2","":"downstairs","session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.631099425Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.631125236Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"d82e9f72-da2d-4e3f-9d48-54779ea98aad is now active with session: 5095ca54-79d3-4a03-b6f7-01eb8918cedd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.631154103Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2026-03-04T01:14:50.631197269Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744,"session_id":"5095ca54-79d3-4a03-b6f7-01eb8918cedd"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.631224372Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744} {"msg":"Activated sub_volume d82e9f72-da2d-4e3f-9d48-54779ea98aad","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:50.631246486Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 488 len 4096 data: 7 02/30 Read block 312 len 12288 03/30 Write block 387 len 40960 data: 8 8 9 10 12 14 12 10 8 11 04/30 Write block 488 len 32768 data: 8 7 8 7 7 6 9 11 05/30 Read block 105 len 16384 06/30 Flush 07/30 Write block 129 len 36864 data: 13 12 10 11 12 14 13 14 14 08/30 Write block 200 len 28672 data: 13 11 10 10 8 12 13 09/30 Read block 310 len 28672 10/30 Write block 377 len 16384 data: 9 10 11 10 11/30 Flush 12/30 Write block 4 len 36864 data: 12 13 15 14 9 10 8 10 9 13/30 Read block 69 len 16384 14/30 Read block 334 len 36864 15/30 Read block 363 len 36864 16/30 Write block 447 len 36864 data: 10 12 13 10 9 7 7 11 9 17/30 Read block 349 len 36864 18/30 Read block 431 len 28672 19/30 Write block 593 len 20480 data: 6 6 7 8 8 20/30 Flush 21/30 Write block 123 len 32768 data: 7 8 8 11 12 11 14 13 22/30 Read block 332 len 12288 23/30 Write block 18 len 12288 data: 8 11 12 24/30 Write block 97 len 16384 data: 18 15 14 14 25/30 Read block 359 len 40960 26/30 Read block 282 len 4096 27/30 Write block 54 len 12288 data: 11 10 12 28/30 Write block 458 len 36864 data: 17 16 16 11 13 13 12 11 13 29/30 Read block 82 len 28672 30/30 Read block 506 len 36864 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 9 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 20 20 Done 24 24 4 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:50.66795299Z","hostname":"bmat-EVT22200007-0000ead9","pid":1744} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 138 138 132 247 247 239 T T F <--- 1 020-039 132 132 132 239 239 239 T T F <--- 2 040-059 128 128 128 235 235 235 T T F <--- 3 060-079 136 136 136 243 243 243 F F F 4 080-099 137 137 137 244 244 244 T T F <--- 5 100-119 137 137 137 244 244 244 T T F <--- 6 120-139 138 138 137 246 246 244 T T T <--- 7 140-159 134 134 134 241 241 241 F F F 8 160-179 129 129 129 236 236 236 F F F 9 180-199 129 129 129 236 236 236 F F F 10 200-219 138 138 133 246 246 240 F F T <--- 11 220-239 137 137 137 244 244 244 F F F 12 240-259 136 136 136 243 243 243 F F F 13 260-279 136 136 136 243 243 243 F F F 14 280-299 134 134 134 241 241 241 F F F 15 300-319 136 136 136 242 242 242 F F F 16 320-339 136 136 136 242 242 242 F F F 17 340-359 134 134 134 241 241 241 F F F 18 360-379 138 138 134 246 246 241 F F T <--- 19 380-399 138 138 138 246 246 245 F F T <--- 20 400-419 136 136 136 242 242 242 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 138 138 136 247 247 243 T T F <--- 23 460-479 131 131 131 238 238 238 T T F <--- 24 480-499 138 138 138 245 245 245 F F F 25 500-519 134 134 134 241 241 241 F F F 26 520-539 134 134 134 241 241 241 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 134 134 134 241 241 241 F F F 29 580-599 138 138 136 247 247 242 F F F <--- Max gen: 138, Max flush: 247 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 248 248 248 F F F 1 020-039 139 139 139 248 248 248 F F F 2 040-059 139 139 139 248 248 248 F F F 3 060-079 136 136 136 243 243 243 F F F 4 080-099 139 139 139 248 248 248 F F F 5 100-119 139 139 139 248 248 248 F F F 6 120-139 139 139 139 248 248 248 F F F 7 140-159 134 134 134 241 241 241 F F F 8 160-179 129 129 129 236 236 236 F F F 9 180-199 129 129 129 236 236 236 F F F 10 200-219 138 138 138 246 246 246 F F F 11 220-239 137 137 137 244 244 244 F F F 12 240-259 136 136 136 243 243 243 F F F 13 260-279 136 136 136 243 243 243 F F F 14 280-299 134 134 134 241 241 241 F F F 15 300-319 136 136 136 242 242 242 F F F 16 320-339 136 136 136 242 242 242 F F F 17 340-359 134 134 134 241 241 241 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 138 138 138 246 246 246 F F F 20 400-419 136 136 136 242 242 242 F F F 21 420-439 137 137 137 244 244 244 F F F 22 440-459 139 139 139 248 248 248 F F F 23 460-479 139 139 139 248 248 248 F F F 24 480-499 138 138 138 245 245 245 F F F 25 500-519 134 134 134 241 241 241 F F F 26 520-539 134 134 134 241 241 241 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 134 134 134 241 241 241 F F F 29 580-599 138 138 138 247 247 247 F F F Max gen: 139, Max flush: 248 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.360446908Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.387655116Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.388329872Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.388475651Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.388511362Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.388560038Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752} {"msg":"Crucible 1f2bc46f-033f-4236-81f7-44f852d657dd has session id: fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.389511933Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"Upstairs opts: Upstairs UUID: 1f2bc46f-033f-4236-81f7-44f852d657dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.38962668Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"Crucible stats registered with UUID: 1f2bc46f-033f-4236-81f7-44f852d657dd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.389661238Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:55.389691067Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.390187701Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.390249974Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.390284081Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.390598694Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.390642651Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752} {"msg":"1f2bc46f-033f-4236-81f7-44f852d657dd active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.390817337Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.390851345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"client":"0","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.390881164Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"client":"1","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.390913087Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"client":"2","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.390958066Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.390991322Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.391162731Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:35409)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.391641008Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:44623)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.391913608Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:37371)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.391969779Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.393575899Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"client":"0","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.393706628Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"client":"2","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.393883658Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"client":"1","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.394421984Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.394775193Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"[0]R flush_numbers[0..12]: [248, 248, 248, 243, 248, 248, 248, 241, 236, 236, 246, 244]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.394852707Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"downstairs","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"[0]R generation[0..12]: [139, 139, 139, 136, 139, 139, 139, 134, 129, 129, 138, 137]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.394887315Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"downstairs","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.394919599Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"downstairs","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"[1]R flush_numbers[0..12]: [248, 248, 248, 243, 248, 248, 248, 241, 236, 236, 246, 244]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.394950691Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"downstairs","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"[1]R generation[0..12]: [139, 139, 139, 136, 139, 139, 139, 134, 129, 129, 138, 137]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.394983476Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"downstairs","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395045007Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"downstairs","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"[2]R flush_numbers[0..12]: [248, 248, 248, 243, 248, 248, 248, 241, 236, 236, 246, 244]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395077261Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"downstairs","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"[2]R generation[0..12]: [139, 139, 139, 136, 139, 139, 139, 134, 129, 129, 138, 137]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395107621Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"downstairs","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395139485Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"downstairs","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395165626Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"downstairs","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.39519254Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"downstairs","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.39521794Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"downstairs","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395243611Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"":"downstairs","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395278149Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395302958Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"client":"0","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395330222Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"client":"1","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395356965Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"client":"2","":"downstairs","session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395384289Z","hostname"Wait for a query_work_queue command to finish before sending IO :"bmat-EVT22200007-0000ead9","pid":1752,"session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395423767Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"1f2bc46f-033f-4236-81f7-44f852d657dd is now active with session: fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395452655Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395476742Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752,"session_id":"fe45e5d2-f1b4-4f43-85e8-8188e8bcc5fd"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395502293Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752} {"msg":"Activated sub_volume 1f2bc46f-033f-4236-81f7-44f852d657dd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:14:55.395524036Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 159 len 20480 data: 9 10 13 12 14 02/30 Write block 459 len 20480 data: 17 17 12 14 14 03/30 Read block 451 len 28672 04/30 Read block 250 len 4096 05/30 Write block 282 len 40960 data: 11 13 15 12 12 12 12 11 11 15 06/30 Read block 131 len 8192 07/30 Write block 423 len 8192 data: 10 11 08/30 Read block 285 len 40960 09/30 Write block 444 len 20480 data: 10 11 12 11 13 10/30 Write block 134 len 8192 data: 15 14 11/30 Write block 78 len 40960 data: 5 5 5 7 9 7 9 9 10 10 12/30 Write block 399 len 40960 data: 10 13 15 13 13 15 14 14 13 13 13/30 Read block 210 len 28672 14/30 Flush 15/30 Write block 523 len 8192 data: 9 8 16/30 Write block 496 len 40960 data: 12 11 10 10 8 7 7 10 10 8 17/30 Flush 18/30 Write block 575 len 16384 data: 12 10 12 15 19/30 Read block 570 len 20480 20/30 Read block 17 len 32768 21/30 Write block 333 len 32768 data: 8 11 9 12 13 11 11 12 22/30 Read block 294 len 16384 23/30 Write block 184 len 32768 data: 10 11 11 12 13 11 11 14 24/30 Read block 319 len 16384 25/30 Read block 74 len 20480 26/30 Write block 238 len 16384 data: 14 16 15 13 27/30 Read block 153 len 28672 28/30 Write block 80 len 36864 data: 6 8 10 8 10 10 11 11 10 29/30 Read block 32 len 24576 30/30 Read block 596 len 12288 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Write 5 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Write 2 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Read 7 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) None Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:14:55.435848773Z","hostname":"bmat-EVT22200007-0000ead9","pid":1752} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 248 248 248 F F F 1 020-039 139 139 139 248 248 248 F F F 2 040-059 139 139 139 248 248 248 F F F 3 060-079 140 136 140 249 243 249 F F F <--- 4 080-099 140 139 140 249 248 249 T F T <--- 5 100-119 139 139 139 248 248 248 F F F 6 120-139 140 139 140 249 248 249 F F F <--- 7 140-159 140 134 140 249 241 249 F F F <--- 8 160-179 140 129 140 249 236 249 F F F <--- 9 180-199 129 129 129 236 236 236 T F T <--- 10 200-219 138 138 138 246 246 246 F F F 11 220-239 137 137 137 244 244 244 T F T <--- 12 240-259 136 136 136 243 243 243 T F T <--- 13 260-279 136 136 136 243 243 243 F F F 14 280-299 140 134 140 249 241 249 F F F <--- 15 300-319 136 136 136 242 242 242 F F F 16 320-339 136 136 136 242 242 242 T F T <--- 17 340-359 134 134 134 241 241 241 T F T <--- 18 360-379 138 138 138 246 246 246 F F F 19 380-399 140 138 140 249 246 249 F F F <--- 20 400-419 140 136 140 249 242 249 F F F <--- 21 420-439 140 137 140 249 244 249 F F F <--- 22 440-459 140 139 140 249 248 249 F F F <--- 23 460-479 140 139 140 249 248 249 F F F <--- 24 480-499 140 138 140 250 245 250 F F F <--- 25 500-519 140 134 140 250 241 250 F F F <--- 26 520-539 140 134 140 250 241 250 F F F <--- 27 540-559 132 132 132 239 239 239 F F F 28 560-579 134 134 134 241 241 241 T F T <--- 29 580-599 138 138 138 247 247 247 F F F Max gen: 140, Max flush: 250 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 248 248 248 F F F 1 020-039 139 139 139 248 248 248 F F F 2 040-059 139 139 139 248 248 248 F F F 3 060-079 140 140 140 249 249 249 F F F 4 080-099 141 141 141 251 251 251 F F F 5 100-119 139 139 139 248 248 248 F F F 6 120-139 140 140 140 249 249 249 F F F 7 140-159 140 140 140 249 249 249 F F F 8 160-179 140 140 140 249 249 249 F F F 9 180-199 141 141 141 251 251 251 F F F 10 200-219 138 138 138 246 246 246 F F F 11 220-239 141 141 141 251 251 251 F F F 12 240-259 141 141 141 251 251 251 F F F 13 260-279 136 136 136 243 243 243 F F F 14 280-299 140 140 140 249 249 249 F F F 15 300-319 136 136 136 242 242 242 F F F 16 320-339 141 141 141 251 251 251 F F F 17 340-359 141 141 141 251 251 251 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 140 140 140 249 249 249 F F F 20 400-419 140 140 140 249 249 249 F F F 21 420-439 140 140 140 249 249 249 F F F 22 440-459 140 140 140 249 249 249 F F F 23 460-479 140 140 140 249 249 249 F F F 24 480-499 140 140 140 250 250 250 F F F 25 500-519 140 140 140 250 250 250 F F F 26 520-539 140 140 140 250 250 250 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 141 141 141 251 251 251 F F F 29 580-599 138 138 138 247 247 247 F F F Max gen: 141, Max flush: 251 Begin loop 71 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.176361201Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:00.200456978Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.20054907Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.225972144Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.226731177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.226842016Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.2268892Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.226930792Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760} {"msg":"Crucible 1acd142b-5a68-4b3f-ab47-29ccddb8f300 has session id: 13961168-5aef-4046-ae7b-a6da7e0792fd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.228301035Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"Upstairs opts: Upstairs UUID: 1acd142b-5a68-4b3f-ab47-29ccddb8f300, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.2283644Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"Crucible stats registered with UUID: 1acd142b-5a68-4b3f-ab47-29ccddb8f300","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.228394069Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:00.22841979Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.228958547Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.229008947Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.229037714Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.229407606Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.229489328Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760} {"msg":"1acd142b-5a68-4b3f-ab47-29ccddb8f300 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.229584316Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.229613293Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"client":"0","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.229640707Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"client":"1","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.22966744Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"client":"2","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.229700365Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.229728211Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.229820994Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:57252)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.229949328Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:50688)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.229988877Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"ds_connection connected from Ok(127.0.0.1:49350)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.230240725Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.232390862Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"client":"2","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.232471241Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"client":"1","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.232734623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"client":"0","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.23343035Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.233484077Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"[0]R flush_numbers[0..12]: [248, 248, 248, 249, 251, 248, 249, 249, 249, 251, 246, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.233623914Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"downstairs","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"[0]R generation[0..12]: [139, 139, 139, 140, 141, 139, 140, 140, 140, 141, 138, 141]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.23366745Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"downstairs","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.233707259Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"downstairs","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"[1]R flush_numbers[0..12]: [248, 248, 248, 249, 251, 248, 249, 249, 249, 251, 246, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.233745544Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"downstairs","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"[1]R generation[0..12]: [139, 139, 139, 140, 141, 139, 140, 140, 140, 141, 138, 141]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.233790443Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"downstairs","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.233829821Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"downstairs","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"[2]R flush_numbers[0..12]: [248, 248, 248, 249, 251, 248, 249, 249, 249, 251, 246, 251]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.233867816Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"downstairs","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"[2]R generation[0..12]: [139, 139, 139, 140, 141, 139, 140, 140, 140, 141, 138, 141]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.233905982Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"downstairs","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.2339453Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"downstairs","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.233977263Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"downstairs","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.23401102Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"downstairs","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.234042783Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"downstairs","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.234074185Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"":"downstairs","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.234107551Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"Wait for a query_work_queue command to finish before sending IO session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.23415252Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"client":"0","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.234185546Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"client":"1","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.23421815Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"client":"2","":"downstairs","session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.234251847Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.234286796Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"1acd142b-5a68-4b3f-ab47-29ccddb8f300 is now active with session: 13961168-5aef-4046-ae7b-a6da7e0792fd","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.234322677Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.234352366Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760,"session_id":"13961168-5aef-4046-ae7b-a6da7e0792fd"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.234383838Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760} {"msg":"Activated sub_volume 1acd142b-5a68-4b3f-ab47-29ccddb8f300","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:00.234410441Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 277 len 16384 data: 13 13 12 12 03/30 Read block 164 len 20480 04/30 Read block 199 len 20480 05/30 Write block 174 len 28672 data: 12 12 15 15 11 14 17 06/30 Read block 166 len 4096 07/30 Write block 2 len 8192 data: 7 10 08/30 Write block 189 len 20480 data: 12 12 15 14 17 09/30 Read block 163 len 16384 10/30 Write block 164 len 32768 data: 13 14 11 12 14 11 10 11 11/30 Write block 216 len 28672 data: 9 11 11 10 12 10 8 12/30 Read block 290 len 32768 13/30 Write block 154 len 20480 data: 8 6 7 8 8 14/30 Read block 309 len 20480 15/30 Read block 271 len 8192 16/30 Write block 492 len 32768 data: 8 7 10 12 13 12 11 11 17/30 Read block 139 len 36864 18/30 Write block 330 len 20480 data: 9 8 8 9 12 19/30 Read block 290 len 24576 20/30 Read block 59 len 32768 21/30 Write block 384 len 40960 data: 8 11 9 9 9 10 11 13 15 13 22/30 Read block 484 len 20480 23/30 Read block 301 len 20480 24/30 Read block 246 len 28672 25/30 Read block 597 len 12288 26/30 Read block 396 len 24576 27/30 Read block 547 len 28672 28/30 Write block 446 len 8192 data: 13 12 29/30 Write block 175 len 32768 data: 13 16 16 12 15 18 12 12 30/30 Write block 41 len 12288 data: 7 7 11 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 4 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 2 2 29 Done 4 27 27 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:00.269313694Z","hostname":"bmat-EVT22200007-0000ead9","pid":1760} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 248 248 248 F T T <--- 1 020-039 139 139 139 248 248 248 F F F 2 040-059 139 139 139 248 248 248 F T T <--- 3 060-079 140 140 140 249 249 249 F F F 4 080-099 141 141 141 251 251 251 F F F 5 100-119 139 139 139 248 248 248 F F F 6 120-139 140 140 140 249 249 249 F F F 7 140-159 140 140 140 249 249 249 F T T <--- 8 160-179 140 140 140 249 249 249 T T T 9 180-199 141 141 141 251 251 251 T T T 10 200-219 138 138 138 246 246 246 F T T <--- 11 220-239 141 141 141 251 251 251 F T T <--- 12 240-259 141 141 141 251 251 251 F F F 13 260-279 136 136 136 243 243 243 T T T 14 280-299 140 140 140 249 249 249 T T T 15 300-319 136 136 136 242 242 242 F F F 16 320-339 141 141 141 251 251 251 F T T <--- 17 340-359 141 141 141 251 251 251 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 140 140 140 249 249 249 F T T <--- 20 400-419 140 140 140 249 249 249 F F F 21 420-439 140 140 140 249 249 249 F F F 22 440-459 140 140 140 249 249 249 F T T <--- 23 460-479 140 140 140 249 249 249 F F F 24 480-499 140 140 140 250 250 250 F T T <--- 25 500-519 140 140 140 250 250 250 F F F 26 520-539 140 140 140 250 250 250 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 141 141 141 251 251 251 F F F 29 580-599 138 138 138 247 247 247 F F F Max gen: 141, Max flush: 251 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 71 Verifying data now Files /tmp/test_live_repair/test_repair_verify.data and /tmp/test_live_repair/test_repair_verify.data.last differ 167d166 < 12, 169c168 < 10, --- > 14, 171c170,172 < 13, --- > 12, > 14, > 11, 172a174 > 11, 174d175 < 10, 176d176 < 9, 177a178,180 > 13, > 16, > 16, 180,185c183,185 < 15, < 11, < 14, < 17, < 11, < 11, --- > 18, > 12, > 12, 192,193c192,194 < 11, < 11, --- > 12, > 12, > 15, 195,196c196 < 13, < 16, --- > 17, diff found after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 252 252 252 F F F 1 020-039 139 139 139 248 248 248 F F F 2 040-059 142 142 142 252 252 252 F F F 3 060-079 140 140 140 249 249 249 F F F 4 080-099 141 141 141 251 251 251 F F F 5 100-119 139 139 139 248 248 248 F F F 6 120-139 140 140 140 249 249 249 F F F 7 140-159 142 142 142 252 252 252 F F F 8 160-179 142 142 142 252 252 252 F F F 9 180-199 142 142 142 252 252 252 F F F 10 200-219 142 142 142 252 252 252 F F F 11 220-239 142 142 142 252 252 252 F F F 12 240-259 141 141 141 251 251 251 F F F 13 260-279 142 142 142 252 252 252 F F F 14 280-299 142 142 142 252 252 252 F F F 15 300-319 136 136 136 242 242 242 F F F 16 320-339 142 142 142 252 252 252 F F F 17 340-359 141 141 141 251 251 251 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 142 142 142 252 252 252 F F F 20 400-419 140 140 140 249 249 249 F F F 21 420-439 140 140 140 249 249 249 F F F 22 440-459 142 142 142 252 252 252 F F F 23 460-479 140 140 140 249 249 249 F F F 24 480-499 142 142 142 252 252 252 F F F 25 500-519 140 140 140 250 250 250 F F F 26 520-539 140 140 140 250 250 250 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 141 141 141 251 251 251 F F F 29 580-599 138 138 138 247 247 247 F F F Max gen: 142, Max flush: 252 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.023239107Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.050239163Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.05094467Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.051090319Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.051132743Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.051183132Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771} {"msg":"Crucible c12491df-afcd-4fb3-9299-7615141809f8 has session id: d0fe7f1b-fbf3-48b6-906c-3040e99c61ef","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.052368259Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"Upstairs opts: Upstairs UUID: c12491df-afcd-4fb3-9299-7615141809f8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.052461253Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"Crucible stats registered with UUID: c12491df-afcd-4fb3-9299-7615141809f8","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.052517214Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:05.052558706Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.053204214Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.053285014Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.053325825Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.053462385Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.053507865Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771} {"msg":"c12491df-afcd-4fb3-9299-7615141809f8 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.053613554Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.053660157Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"client":"0","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.053695687Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"client":"1","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.05372749Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"client":"2","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.053761517Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.053796216Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.053918398Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:46852)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.054287589Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"2","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:64921)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.054342548Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"0","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:05.054388829Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:05.054446774Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:05.054490621Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.054724885Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.05476778Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.056591891Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"client":"0","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.057024868Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"client":"2","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.057533806Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:05.058102331Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.055756091Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:63982)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.056087246Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"io task","client":"1","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.057936528Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"client":"1","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"[0]R flush_numbers[0..12]: [252, 248, 252, 249, 251, 248, 249, 252, 252, 252, 252, 252]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.058970595Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"downstairs","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"[0]R generation[0..12]: [142, 139, 142, 140, 141, 139, 140, 142, 142, 142, 142, 142]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059024081Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"downstairs","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059072797Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"downstairs","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"[1]R flush_numbers[0..12]: [252, 248, 252, 249, 251, 248, 249, 252, 252, 252, 252, 252]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059122195Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"downstairs","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"[1]R generation[0..12]: [142, 139, 142, 140, 141, 139, 140, 142, 142, 142, 142, 142]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059169358Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"downstairs","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059209348Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"downstairs","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"[2]R flush_numbers[0..12]: [252, 248, 252, 249, 251, 248, 249, 252, 252, 252, 252, 252]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059241601Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"downstairs","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"[2]R generation[0..12]: [142, 139, 142, 140, 141, 139, 140, 142, 142, 142, 142, 142]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059271811Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"downstairs","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059319706Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"downstairs","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"Max found gen is 143","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059352681Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"downstairs","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"Generation requested: 144 >= found:143","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059380466Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"downstairs","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059420085Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"downstairs","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.05945306Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"":"downstairs","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059480484Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059505083Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"client":"0","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059533229Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"client":"1","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.05957403Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"client":"2","":"downstairs","session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.05960958Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059647184Z","hostname":"Wait for a query_work_queue command to finish before sending IO bmat-EVT22200007-0000ead9","pid":1771,"session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"c12491df-afcd-4fb3-9299-7615141809f8 is now active with session: d0fe7f1b-fbf3-48b6-906c-3040e99c61ef","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059702614Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059728736Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771,"session_id":"d0fe7f1b-fbf3-48b6-906c-3040e99c61ef"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059754246Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771} {"msg":"Activated sub_volume c12491df-afcd-4fb3-9299-7615141809f8","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:15.059776621Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 480 len 36864 data: 10 9 10 10 8 8 8 9 9 02/30 Read block 565 len 4096 03/30 Write block 573 len 36864 data: 8 10 13 11 13 16 15 17 16 04/30 Write block 497 len 12288 data: 13 12 12 05/30 Flush 06/30 Flush 07/30 Write block 150 len 24576 data: 8 10 7 8 9 7 08/30 Read block 138 len 32768 09/30 Read block 558 len 40960 10/30 Read block 520 len 40960 11/30 Write block 571 len 40960 data: 11 11 9 11 14 12 14 17 16 18 12/30 Write block 444 len 24576 data: 11 12 14 13 14 14 13/30 Flush 14/30 Write block 514 len 8192 data: 10 9 15/30 Flush 16/30 Write block 383 len 28672 data: 7 9 12 10 10 10 11 17/30 Read block 311 len 4096 18/30 Read block 61 len 12288 19/30 Write block 227 len 40960 data: 8 9 9 10 11 11 13 15 13 15 20/30 Read block 388 len 40960 21/30 Read block 375 len 32768 22/30 Read block 8 len 20480 23/30 Read block 253 len 36864 24/30 Read block 213 len 36864 25/30 Write block 352 len 4096 data: 5 26/30 Read block 323 len 24576 27/30 Read block 264 len 8192 28/30 Read block 325 len 24576 29/30 Read block 434 len 24576 30/30 Write block 462 len 16384 data: 15 15 13 12 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 6 Done Done Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 5 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 23 1 25 Done 23 1 23 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1005)) Some(JobId(1014)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:15.101209091Z","hostname":"bmat-EVT22200007-0000ead9","pid":1771} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 252 252 252 F F F 1 020-039 139 139 139 248 248 248 F F F 2 040-059 142 142 142 252 252 252 F F F 3 060-079 140 140 140 249 249 249 F F F 4 080-099 141 141 141 251 251 251 F F F 5 100-119 139 139 139 248 248 248 F F F 6 120-139 140 140 140 249 249 249 F F F 7 140-159 144 142 144 255 252 255 F T F <--- 8 160-179 142 142 142 252 252 252 F F F 9 180-199 142 142 142 252 252 252 F F F 10 200-219 142 142 142 252 252 252 F F F 11 220-239 142 142 142 252 252 252 T F T <--- 12 240-259 141 141 141 251 251 251 F F F 13 260-279 142 142 142 252 252 252 F F F 14 280-299 142 142 142 252 252 252 F F F 15 300-319 136 136 136 242 242 242 F F F 16 320-339 142 142 142 252 252 252 F F F 17 340-359 141 141 141 251 251 251 T F T <--- 18 360-379 138 138 138 246 246 246 F F F 19 380-399 142 142 142 252 252 252 T F T <--- 20 400-419 140 140 140 249 249 249 F F F 21 420-439 140 140 140 249 249 249 F F F 22 440-459 144 142 144 255 252 255 F F F <--- 23 460-479 140 140 140 249 249 249 T F T <--- 24 480-499 144 144 144 253 253 253 F F F 25 500-519 144 140 144 256 250 256 F F F <--- 26 520-539 140 140 140 250 250 250 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 144 144 144 255 253 255 F F F <--- 29 580-599 144 144 144 255 253 255 F F F <--- Max gen: 144, Max flush: 256 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 252 252 252 F F F 1 020-039 139 139 139 248 248 248 F F F 2 040-059 142 142 142 252 252 252 F F F 3 060-079 140 140 140 249 249 249 F F F 4 080-099 141 141 141 251 251 251 F F F 5 100-119 139 139 139 248 248 248 F F F 6 120-139 140 140 140 249 249 249 F F F 7 140-159 144 144 144 255 255 255 F F F 8 160-179 142 142 142 252 252 252 F F F 9 180-199 142 142 142 252 252 252 F F F 10 200-219 142 142 142 252 252 252 F F F 11 220-239 145 145 145 257 257 257 F F F 12 240-259 141 141 141 251 251 251 F F F 13 260-279 142 142 142 252 252 252 F F F 14 280-299 142 142 142 252 252 252 F F F 15 300-319 136 136 136 242 242 242 F F F 16 320-339 142 142 142 252 252 252 F F F 17 340-359 145 145 145 257 257 257 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 145 145 145 257 257 257 F F F 20 400-419 140 140 140 249 249 249 F F F 21 420-439 140 140 140 249 249 249 F F F 22 440-459 144 144 144 255 255 255 F F F 23 460-479 145 145 145 257 257 257 F F F 24 480-499 144 144 144 253 253 253 F F F 25 500-519 144 144 144 256 256 256 F F F 26 520-539 140 140 140 250 250 250 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 144 144 144 255 255 255 F F F 29 580-599 144 144 144 255 255 255 F F F Max gen: 145, Max flush: 257 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.716444917Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:19.745575342Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.745670901Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.770600698Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.771208552Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.771329501Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.771356034Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.771391915Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779} {"msg":"Crucible 65f10d8f-3bde-4d81-9ad9-60a6dbc8fa4e has session id: d8f99554-3272-4743-920b-d98d3ca7fd65","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.772449068Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"Upstairs opts: Upstairs UUID: 65f10d8f-3bde-4d81-9ad9-60a6dbc8fa4e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.772678292Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"Crucible stats registered with UUID: 65f10d8f-3bde-4d81-9ad9-60a6dbc8fa4e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.77272302Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:19.772755885Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.773227219Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.773282809Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.773311706Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.773473115Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.773502303Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779} {"msg":"65f10d8f-3bde-4d81-9ad9-60a6dbc8fa4e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.773690255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.773845843Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"client":"0","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.773877426Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"client":"1","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.774319381Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"client":"2","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.774477454Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.774629435Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.774676909Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"ds_connection connected from Ok(127.0.0.1:53093)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.774725154Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"io task","client":"1","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"ds_connection connected from Ok(127.0.0.1:56286)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.77477357Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"io task","client":"0","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"ds_connection connected from Ok(127.0.0.1:49788)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.774821695Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"io task","client":"2","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.776208741Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"client":"2","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.776327627Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"client":"1","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.776909098Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"client":"0","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.776956211Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.777099896Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"[0]R flush_numbers[0..12]: [252, 248, 252, 249, 251, 248, 249, 255, 252, 252, 252, 257]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.777677179Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"downstairs","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"[0]R generation[0..12]: [142, 139, 142, 140, 141, 139, 140, 144, 142, 142, 142, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.777716837Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"downstairs","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.777749181Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"downstairs","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"[1]R flush_numbers[0..12]: [252, 248, 252, 249, 251, 248, 249, 255, 252, 252, 252, 257]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.777780032Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"downstairs","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"[1]R generation[0..12]: [142, 139, 142, 140, 141, 139, 140, 144, 142, 142, 142, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.777810413Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"downstairs","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.777843107Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"downstairs","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"[2]R flush_numbers[0..12]: [252, 248, 252, 249, 251, 248, 249, 255, 252, 252, 252, 257]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.777874119Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"downstairs","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"[2]R generation[0..12]: [142, 139, 142, 140, 141, 139, 140, 144, 142, 142, 142, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.77790489Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"downstairs","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.777936663Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"downstairs","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.777968075Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"downstairs","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.777995689Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"downstairs","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.778022192Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"downstairs","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.778047732Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"":"downstairs","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.778074546Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-04T01:15:19.778099214Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"client":"0","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.778138973Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"client":"1","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.778166047Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"client":"2","":"downstairs","session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.778193321Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.778218891Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"65f10d8f-3bde-4d81-9ad9-60a6dbc8fa4e is now active with session: d8f99554-3272-4743-920b-d98d3ca7fd65","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.778247839Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.778272417Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779,"session_id":"d8f99554-3272-4743-920b-d98d3ca7fd65"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.778297687Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779} {"msg":"Activated sub_volume 65f10d8f-3bde-4d81-9ad9-60a6dbc8fa4e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:19.778319561Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 235 len 28672 data: 14 16 13 15 17 16 14 03/30 Read block 384 len 40960 04/30 Read block 505 len 8192 05/30 Flush 06/30 Write block 159 len 24576 data: 10 11 14 13 15 13 07/30 Flush 08/30 Write block 151 len 12288 data: 11 8 9 09/30 Read block 36 len 32768 10/30 Read block 597 len 8192 11/30 Flush 12/30 Write block 113 len 32768 data: 17 15 14 15 17 15 13 14 13/30 Write block 345 len 24576 data: 9 10 9 8 8 8 14/30 Flush 15/30 Write block 349 len 20480 data: 9 9 7 6 6 16/30 Flush 17/30 Write block 435 len 36864 data: 13 16 19 18 14 13 11 12 9 18/30 Read block 164 len 40960 19/30 Read block 212 len 36864 20/30 Write block 435 len 24576 data: 14 17 20 19 15 14 21/30 Read block 383 len 4096 22/30 Flush 23/30 Write block 49 len 40960 data: 10 9 11 11 11 12 11 13 10 5 24/30 Write block 10 len 24576 data: 9 11 10 9 8 8 25/30 Write block 382 len 8192 data: 9 8 26/30 Flush 27/30 Write block 410 len 8192 data: 10 9 28/30 Read block 147 len 20480 29/30 Write block 253 len 12288 data: 8 9 10 30/30 Write block 5 len 12288 data: 14 16 15 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Sent Done Done false 1001 Acked Write 7 Sent Done Done false 1002 Acked Read 10 Sent Done Done false 1003 Acked Read 2 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:19.814050673Z","hostname":"bmat-EVT22200007-0000ead9","pid":1779} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 146 146 252 265 265 F T T <--- 1 020-039 139 139 139 248 248 248 F F F 2 040-059 142 146 146 252 265 265 F F F <--- 3 060-079 140 140 140 249 249 249 F F F 4 080-099 141 141 141 251 251 251 F F F 5 100-119 139 146 146 248 262 262 F F F <--- 6 120-139 140 146 146 249 262 262 F F F <--- 7 140-159 144 146 146 255 261 261 F F F <--- 8 160-179 142 146 146 252 260 260 F F F <--- 9 180-199 142 142 142 252 252 252 F F F 10 200-219 142 142 142 252 252 252 F F F 11 220-239 145 146 146 257 259 259 F F F <--- 12 240-259 141 146 146 251 259 259 F T T <--- 13 260-279 142 142 142 252 252 252 F F F 14 280-299 142 142 142 252 252 252 F F F 15 300-319 136 136 136 242 242 242 F F F 16 320-339 142 142 142 252 252 252 F F F 17 340-359 145 146 146 257 263 263 F F F <--- 18 360-379 138 138 138 246 246 246 F F F 19 380-399 145 146 146 257 265 265 F F F <--- 20 400-419 140 140 140 249 249 249 F T T <--- 21 420-439 140 146 146 249 264 264 F F F <--- 22 440-459 144 146 146 255 264 264 F F F <--- 23 460-479 145 145 145 257 257 257 F F F 24 480-499 144 144 144 253 253 253 F F F 25 500-519 144 144 144 256 256 256 F F F 26 520-539 140 140 140 250 250 250 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 144 144 144 255 255 255 F F F 29 580-599 144 144 144 255 255 255 F F F Max gen: 146, Max flush: 265 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 F F F 1 020-039 139 139 139 248 248 248 F F F 2 040-059 146 146 146 265 265 265 F F F 3 060-079 140 140 140 249 249 249 F F F 4 080-099 141 141 141 251 251 251 F F F 5 100-119 146 146 146 262 262 262 F F F 6 120-139 146 146 146 262 262 262 F F F 7 140-159 146 146 146 261 261 261 F F F 8 160-179 146 146 146 260 260 260 F F F 9 180-199 142 142 142 252 252 252 F F F 10 200-219 142 142 142 252 252 252 F F F 11 220-239 146 146 146 259 259 259 F F F 12 240-259 147 147 147 266 266 266 F F F 13 260-279 142 142 142 252 252 252 F F F 14 280-299 142 142 142 252 252 252 F F F 15 300-319 136 136 136 242 242 242 F F F 16 320-339 142 142 142 252 252 252 F F F 17 340-359 146 146 146 263 263 263 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 146 146 146 265 265 265 F F F 20 400-419 147 147 147 266 266 266 F F F 21 420-439 146 146 146 264 264 264 F F F 22 440-459 146 146 146 264 264 264 F F F 23 460-479 145 145 145 257 257 257 F F F 24 480-499 144 144 144 253 253 253 F F F 25 500-519 144 144 144 256 256 256 F F F 26 520-539 140 140 140 250 250 250 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 144 144 144 255 255 255 F F F 29 580-599 144 144 144 255 255 255 F F F Max gen: 147, Max flush: 266 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.477179302Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.504909444Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.505437349Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.505556505Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.505600653Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.505651563Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787} {"msg":"Crucible 11ca13d3-e6f2-40c5-a761-cc2592e6ca42 has session id: 492e1ed2-fe0f-439b-a721-8dd96654bc71","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.506749157Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"Upstairs opts: Upstairs UUID: 11ca13d3-e6f2-40c5-a761-cc2592e6ca42, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.506825237Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"Crucible stats registered with UUID: 11ca13d3-e6f2-40c5-a761-cc2592e6ca42","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.506874264Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:24.50691242Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.507222454Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.507302272Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.507332121Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.507637165Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"task":"crutest"} {"msg":"11ca13d3-e6f2-40c5-a761-cc2592e6ca42 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.508034682Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.508067667Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"client":"0","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.508096805Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.508117436Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"client":"1","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.508151884Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"client":"2","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.508179148Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.508213927Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.508242894Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"ds_connection connected from Ok(127.0.0.1:59863)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.508447128Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"0","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"ds_connection connected from Ok(127.0.0.1:49642)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.508481536Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"2","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:24.508729036Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:24.508773555Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:24.508925786Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.509078979Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.509239497Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.510399654Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"client":"2","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.510506746Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"client":"0","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.511369204Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:24.51143323Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.510896061Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"ds_connection connected from Ok(127.0.0.1:34900)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.511286324Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"io task","client":"1","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.514186376Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"client":"1","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"[0]R flush_numbers[0..12]: [266, 248, 265, 249, 251, 262, 262, 261, 260, 252, 252, 259]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516152338Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"downstairs","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"[0]R generation[0..12]: [147, 139, 146, 140, 141, 146, 146, 146, 146, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.51622892Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"downstairs","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516276624Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"downstairs","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"[1]R flush_numbers[0..12]: [266, 248, 265, 249, 251, 262, 262, 261, 260, 252, 252, 259]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516323326Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"downstairs","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"[1]R generation[0..12]: [147, 139, 146, 140, 141, 146, 146, 146, 146, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516369127Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"downstairs","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516417653Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"downstairs","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"[2]R flush_numbers[0..12]: [266, 248, 265, 249, 251, 262, 262, 261, 260, 252, 252, 259]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516462582Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"downstairs","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"[2]R generation[0..12]: [147, 139, 146, 140, 141, 146, 146, 146, 146, 142, 142, 146]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516500447Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"downstairs","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516538532Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"downstairs","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516568291Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"downstairs","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516608431Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"downstairs","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.51663826Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"downstairs","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-04T01:15:34.516666977Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"":"downstairs","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.51670903Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516742657Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"client":"0","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516777886Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"client":"1","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516810301Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"client":"2","":"downstairs","session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516842875Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.51687558Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"11ca13d3-e6f2-40c5-a761-cc2592e6ca42 is now active with session: 492e1ed2-fe0f-439b-a721-8dd96654bc71","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516910649Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-04T01:15:34.516944005Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787,"session_id":"492e1ed2-fe0f-439b-a721-8dd96654bc71"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.516984385Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787} {"msg":"Activated sub_volume 11ca13d3-e6f2-40c5-a761-cc2592e6ca42","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:34.517016168Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 227 len 8192 data: 9 10 02/30 Read block 257 len 40960 03/30 Read block 407 len 8192 04/30 Read block 248 len 4096 05/30 Write block 415 len 16384 data: 10 9 6 5 06/30 Read block 245 len 4096 07/30 Read block 163 len 20480 08/30 Write block 536 len 12288 data: 14 12 12 09/30 Read block 106 len 8192 10/30 Read block 30 len 20480 11/30 Write block 84 len 16384 data: 11 11 12 12 12/30 Write block 530 len 40960 data: 10 10 11 14 14 12 15 13 13 14 13/30 Write block 447 len 8192 data: 14 15 14/30 Flush 15/30 Write block 569 len 20480 data: 11 14 12 12 10 16/30 Write block 479 len 16384 data: 9 11 10 11 17/30 Read block 498 len 40960 18/30 Write block 274 len 40960 data: 14 13 13 14 14 13 13 12 12 14 19/30 Flush 20/30 Write block 190 len 12288 data: 12 15 14 21/30 Write block 592 len 24576 data: 8 7 7 8 9 9 22/30 Write block 442 len 36864 data: 13 10 12 13 15 15 16 15 11 23/30 Write block 154 len 28672 data: 10 8 8 9 9 11 12 24/30 Read block 85 len 32768 25/30 Read block 404 len 32768 26/30 Read block 590 len 12288 27/30 Write block 115 len 16384 data: 15 16 18 16 28/30 Read block 126 len 12288 29/30 Read block 311 len 24576 30/30 Write block 411 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:34.548478219Z","hostname":"bmat-EVT22200007-0000ead9","pid":1787} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 F F F 1 020-039 139 139 139 248 248 248 F F F 2 040-059 146 146 146 265 265 265 F F F 3 060-079 140 140 140 249 249 249 F F F 4 080-099 148 141 148 267 251 267 F F F <--- 5 100-119 146 146 146 262 262 262 T F T <--- 6 120-139 146 146 146 262 262 262 F F F 7 140-159 146 146 146 261 261 261 T F T <--- 8 160-179 146 146 146 260 260 260 T F T <--- 9 180-199 142 142 142 252 252 252 T F T <--- 10 200-219 142 142 142 252 252 252 F F F 11 220-239 148 146 148 267 259 267 F T F <--- 12 240-259 147 147 147 266 266 266 F F F 13 260-279 148 142 148 268 252 268 F F F <--- 14 280-299 148 142 148 268 252 268 F F F <--- 15 300-319 136 136 136 242 242 242 F F F 16 320-339 142 142 142 252 252 252 F F F 17 340-359 146 146 146 263 263 263 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 146 146 146 265 265 265 F F F 20 400-419 148 147 148 267 266 267 T F T <--- 21 420-439 146 146 146 264 264 264 F F F 22 440-459 148 146 148 267 264 267 T F T <--- 23 460-479 148 145 148 268 257 268 F F F <--- 24 480-499 148 144 148 268 253 268 F F F <--- 25 500-519 144 144 144 256 256 256 F F F 26 520-539 148 140 148 267 250 267 F F F <--- 27 540-559 132 132 132 239 239 239 F F F 28 560-579 148 144 148 268 255 268 F F F <--- 29 580-599 144 144 144 255 255 255 T F T <--- Max gen: 148, Max flush: 268 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 F F F 1 020-039 139 139 139 248 248 248 F F F 2 040-059 146 146 146 265 265 265 F F F 3 060-079 140 140 140 249 249 249 F F F 4 080-099 148 148 148 267 267 267 F F F 5 100-119 149 149 149 269 269 269 F F F 6 120-139 146 146 146 262 262 262 F F F 7 140-159 149 149 149 269 269 269 F F F 8 160-179 149 149 149 269 269 269 F F F 9 180-199 149 149 149 269 269 269 F F F 10 200-219 142 142 142 252 252 252 F F F 11 220-239 148 148 148 267 267 267 F F F 12 240-259 147 147 147 266 266 266 F F F 13 260-279 148 148 148 268 268 268 F F F 14 280-299 148 148 148 268 268 268 F F F 15 300-319 136 136 136 242 242 242 F F F 16 320-339 142 142 142 252 252 252 F F F 17 340-359 146 146 146 263 263 263 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 146 146 146 265 265 265 F F F 20 400-419 149 149 149 269 269 269 F F F 21 420-439 146 146 146 264 264 264 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 148 148 148 268 268 268 F F F 24 480-499 148 148 148 268 268 268 F F F 25 500-519 144 144 144 256 256 256 F F F 26 520-539 148 148 148 267 267 267 F F F 27 540-559 132 132 132 239 239 239 F F F 28 560-579 148 148 148 268 268 268 F F F 29 580-599 149 149 149 269 269 269 F F F Max gen: 149, Max flush: 269 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.224018601Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.25216164Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.252789674Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.252882187Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.252925974Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.252975642Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795} {"msg":"Crucible 0ccae7ee-2b06-4869-b20c-ae4aaf6178f2 has session id: 0095f688-8a59-411e-b4ae-a80894c1b899","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.253765606Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"Upstairs opts: Upstairs UUID: 0ccae7ee-2b06-4869-b20c-ae4aaf6178f2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.253844853Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"Crucible stats registered with UUID: 0ccae7ee-2b06-4869-b20c-ae4aaf6178f2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.253885083Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:39.253919591Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.254467015Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.25454052Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.254580479Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.25462138Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.254671069Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795} {"msg":"0ccae7ee-2b06-4869-b20c-ae4aaf6178f2 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.254847288Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.254885603Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"client":"0","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.254923609Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"client":"1","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.254968017Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"client":"2","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.255005Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.255043386Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.255092263Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"ds_connection connected from Ok(127.0.0.1:61072)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.255312839Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"io task","client":"0","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"ds_connection connected from Ok(127.0.0.1:52005)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.255360824Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"io task","client":"2","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"ds_connection connected from Ok(127.0.0.1:41786)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.255401575Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"io task","client":"1","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.256922947Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"client":"0","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.257023536Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"client":"2","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.257593715Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"client":"1","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.257648754Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.257834291Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"[0]R flush_numbers[0..12]: [266, 248, 265, 249, 267, 269, 262, 269, 269, 269, 252, 267]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.258694114Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"downstairs","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"[0]R generation[0..12]: [147, 139, 146, 140, 148, 149, 146, 149, 149, 149, 142, 148]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.25874272Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"downstairs","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.258775956Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"downstairs","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"[1]R flush_numbers[0..12]: [266, 248, 265, 249, 267, 269, 262, 269, 269, 269, 252, 267]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.2588134Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"downstairs","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"[1]R generation[0..12]: [147, 139, 146, 140, 148, 149, 146, 149, 149, 149, 142, 148]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.258844121Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"downstairs","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.258875673Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"downstairs","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"[2]R flush_numbers[0..12]: [266, 248, 265, 249, 267, 269, 262, 269, 269, 269, 252, 267]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.258906043Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"downstairs","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"[2]R generation[0..12]: [147, 139, 146, 140, 148, 149, 146, 149, 149, 149, 142, 148]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.258936814Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"downstairs","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.258968988Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"downstairs","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.258994879Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"downstairs","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.259022003Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"downstairs","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.259047343Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"downstairs","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.259072804Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"":"downstairs","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.259099727Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.259124035Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"client":"0","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.259150698Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"client":"1","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-04T01:15:39.25917717Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"client":"2","":"downstairs","session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.259221668Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.259250335Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"0ccae7ee-2b06-4869-b20c-ae4aaf6178f2 is now active with session: 0095f688-8a59-411e-b4ae-a80894c1b899","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.259279092Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.25930316Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795,"session_id":"0095f688-8a59-411e-b4ae-a80894c1b899"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.25932849Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795} {"msg":"Activated sub_volume 0ccae7ee-2b06-4869-b20c-ae4aaf6178f2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:39.259350383Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 79 len 40960 data: 6 7 9 11 9 12 12 13 13 11 02/30 Flush 03/30 Write block 139 len 4096 data: 17 04/30 Read block 244 len 4096 05/30 Read block 592 len 12288 06/30 Read block 543 len 36864 07/30 Write block 178 len 28672 data: 12 15 18 12 12 10 11 08/30 Write block 108 len 32768 data: 8 7 10 11 11 18 16 16 09/30 Read block 387 len 8192 10/30 Read block 301 len 4096 11/30 Read block 58 len 4096 12/30 Write block 540 len 36864 data: 12 12 14 12 10 11 9 10 9 13/30 Write block 207 len 16384 data: 15 15 16 14 14/30 Write block 486 len 28672 data: 9 10 10 8 9 8 9 15/30 Read block 535 len 36864 16/30 Read block 261 len 32768 17/30 Read block 61 len 36864 18/30 Read block 305 len 40960 19/30 Read block 195 len 16384 20/30 Write block 531 len 32768 data: 11 12 15 15 13 16 14 14 21/30 Write block 44 len 36864 data: 11 14 11 10 8 11 10 12 12 22/30 Write block 542 len 4096 data: 15 23/30 Write block 265 len 36864 data: 10 12 9 12 12 14 14 14 13 24/30 Read block 300 len 28672 25/30 Write block 303 len 4096 data: 11 26/30 Write block 512 len 32768 data: 10 9 11 10 8 8 7 8 27/30 Read block 244 len 40960 28/30 Write block 36 len 16384 data: 10 9 8 8 29/30 Write block 7 len 8192 data: 16 10 30/30 Read block 44 len 4096 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 1 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Read 1 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 24 0 25 Done 27 4 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:39.294314628Z","hostname":"bmat-EVT22200007-0000ead9","pid":1795} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 147 147 147 266 266 266 T F T <--- 1 020-039 139 139 139 248 248 248 T F T <--- 2 040-059 146 146 146 265 265 265 T F T <--- 3 060-079 150 150 150 270 270 270 F F F 4 080-099 150 150 150 270 270 270 F F F 5 100-119 149 149 149 269 269 269 T F T <--- 6 120-139 146 146 146 262 262 262 T T T 7 140-159 149 149 149 269 269 269 F F F 8 160-179 149 149 149 269 269 269 T F T <--- 9 180-199 149 149 149 269 269 269 T F T <--- 10 200-219 142 142 142 252 252 252 T F T <--- 11 220-239 148 148 148 267 267 267 F F F 12 240-259 147 147 147 266 266 266 F F F 13 260-279 148 148 148 268 268 268 T F T <--- 14 280-299 148 148 148 268 268 268 F F F 15 300-319 136 136 136 242 242 242 T F T <--- 16 320-339 142 142 142 252 252 252 F F F 17 340-359 146 146 146 263 263 263 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 146 146 146 265 265 265 F F F 20 400-419 149 149 149 269 269 269 F F F 21 420-439 146 146 146 264 264 264 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 148 148 148 268 268 268 F F F 24 480-499 148 148 148 268 268 268 T F T <--- 25 500-519 144 144 144 256 256 256 T F T <--- 26 520-539 148 148 148 267 267 267 T F T <--- 27 540-559 132 132 132 239 239 239 T F T <--- 28 560-579 148 148 148 268 268 268 F F F 29 580-599 149 149 149 269 269 269 F F F Max gen: 150, Max flush: 270 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 271 271 271 F F F 1 020-039 151 151 151 271 271 271 F F F 2 040-059 151 151 151 271 271 271 F F F 3 060-079 150 150 150 270 270 270 F F F 4 080-099 150 150 150 270 270 270 F F F 5 100-119 151 151 151 271 271 271 F F F 6 120-139 151 151 151 271 271 271 F F F 7 140-159 149 149 149 269 269 269 F F F 8 160-179 151 151 151 271 271 271 F F F 9 180-199 151 151 151 271 271 271 F F F 10 200-219 151 151 151 271 271 271 F F F 11 220-239 148 148 148 267 267 267 F F F 12 240-259 147 147 147 266 266 266 F F F 13 260-279 151 151 151 271 271 271 F F F 14 280-299 148 148 148 268 268 268 F F F 15 300-319 151 151 151 271 271 271 F F F 16 320-339 142 142 142 252 252 252 F F F 17 340-359 146 146 146 263 263 263 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 146 146 146 265 265 265 F F F 20 400-419 149 149 149 269 269 269 F F F 21 420-439 146 146 146 264 264 264 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 148 148 148 268 268 268 F F F 24 480-499 151 151 151 271 271 271 F F F 25 500-519 151 151 151 271 271 271 F F F 26 520-539 151 151 151 271 271 271 F F F 27 540-559 151 151 151 271 271 271 F F F 28 560-579 148 148 148 268 268 268 F F F 29 580-599 149 149 149 269 269 269 F F F Max gen: 151, Max flush: 271 Begin loop 76 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.006454001Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.03486486Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.035860872Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.035972714Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.036013635Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.036069014Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803} {"msg":"Crucible f2ffc85f-9220-46a9-ac73-c3d226a9b956 has session id: 09c0b151-59f4-454c-a66c-9c40f16780e9","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.03726878Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"Upstairs opts: Upstairs UUID: f2ffc85f-9220-46a9-ac73-c3d226a9b956, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.037367866Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"Crucible stats registered with UUID: f2ffc85f-9220-46a9-ac73-c3d226a9b956","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.037421652Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:44.037485298Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.037928706Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.038030378Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.038065607Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.038471351Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.038519095Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803} {"msg":"f2ffc85f-9220-46a9-ac73-c3d226a9b956 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.038646377Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.03868297Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"client":"0","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.038713751Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"client":"1","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.038745012Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"client":"2","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.038774441Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.038805332Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.038982974Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:59245)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.039184342Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"io task","client":"2","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:64707)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.03922345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"io task","client":"0","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:62455)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.039772897Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"io task","client":"1","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.041222978Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"client":"2","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.041477132Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"client":"0","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.041558252Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"client":"1","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.04219859Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042246475Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"[0]R flush_numbers[0..12]: [271, 271, 271, 270, 270, 271, 271, 269, 271, 271, 271, 267]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042280602Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"downstairs","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"[0]R generation[0..12]: [151, 151, 151, 150, 150, 151, 151, 149, 151, 151, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042315451Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"downstairs","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.04235032Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"downstairs","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"[1]R flush_numbers[0..12]: [271, 271, 271, 270, 270, 271, 271, 269, 271, 271, 271, 267]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042383897Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"downstairs","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"[1]R generation[0..12]: [151, 151, 151, 150, 150, 151, 151, 149, 151, 151, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042417814Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"downstairs","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042453144Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"downstairs","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"[2]R flush_numbers[0..12]: [271, 271, 271, 270, 270, 271, 271, 269, 271, 271, 271, 267]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042487392Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"downstairs","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"[2]R generation[0..12]: [151, 151, 151, 150, 150, 151, 151, 149, 151, 151, 151, 148]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042520467Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"downstairs","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042555126Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"downstairs","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042583532Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"downstairs","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042620185Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"downstairs","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042648521Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"downstairs","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042676536Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"":"downstairs","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042705924Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042732918Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"client":"0","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042762757Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"client":"1","":"downstairs","session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042792185Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"client":"2","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.04284501Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042873737Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"f2ffc85f-9220-46a9-ac73-c3d226a9b956 is now active with session: 09c0b151-59f4-454c-a66c-9c40f16780e9","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.04290547Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042931922Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803,"session_id":"09c0b151-59f4-454c-a66c-9c40f16780e9"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042959707Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803} {"msg":"Activated sub_volume f2ffc85f-9220-46a9-ac73-c3d226a9b956","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:44.042983644Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 157 len 40960 data: 10 10 12 13 15 14 16 14 14 11 02/30 Read block 313 len 36864 03/30 Read block 423 len 40960 04/30 Read block 385 len 36864 05/30 Write block 288 len 36864 data: 13 12 12 16 14 9 10 11 10 06/30 Write block 562 len 24576 data: 11 12 12 10 10 12 07/30 Read block 313 len 20480 08/30 Write block 324 len 24576 data: 13 8 8 7 8 8 09/30 Read block 376 len 36864 10/30 Read block 481 len 8192 11/30 Write block 126 len 24576 data: 12 13 12 15 14 11 12/30 Read block 352 len 20480 13/30 Read block 55 len 40960 14/30 Read block 479 len 8192 15/30 Write block 200 len 28672 data: 14 12 11 11 9 13 14 16/30 Write block 202 len 32768 data: 12 12 10 14 15 16 16 17 17/30 Read block 16 len 24576 18/30 Read block 207 len 36864 19/30 Read block 34 len 12288 20/30 Write block 501 len 28672 data: 8 8 11 11 9 7 7 21/30 Write block 547 len 8192 data: 11 10 22/30 Write block 549 len 8192 data: 7 9 23/30 Write block 294 len 40960 data: 11 12 11 10 10 8 9 8 10 12 24/30 Read block 355 len 4096 25/30 Write block 309 len 32768 data: 14 14 13 9 7 5 7 10 26/30 Read block 587 len 20480 27/30 Read block 26 len 4096 28/30 Write block 326 len 4096 data: 9 29/30 Read block 202 len 12288 30/30 Write block 197 len 12288 data: 12 13 11 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Sent Done false 1001 Acked Read 9 Done Sent Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:44.080614438Z","hostname":"bmat-EVT22200007-0000ead9","pid":1803} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 271 271 271 F F F 1 020-039 151 151 151 271 271 271 F F F 2 040-059 151 151 151 271 271 271 F F F 3 060-079 150 150 150 270 270 270 F F F 4 080-099 150 150 150 270 270 270 F F F 5 100-119 151 151 151 271 271 271 F F F 6 120-139 151 151 151 271 271 271 T F T <--- 7 140-159 149 149 149 269 269 269 T F T <--- 8 160-179 151 151 151 271 271 271 T F T <--- 9 180-199 151 151 151 271 271 271 T F T <--- 10 200-219 151 151 151 271 271 271 T F T <--- 11 220-239 148 148 148 267 267 267 F F F 12 240-259 147 147 147 266 266 266 F F F 13 260-279 151 151 151 271 271 271 F F F 14 280-299 148 148 148 268 268 268 T F T <--- 15 300-319 151 151 151 271 271 271 T F T <--- 16 320-339 142 142 142 252 252 252 T F T <--- 17 340-359 146 146 146 263 263 263 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 146 146 146 265 265 265 F F F 20 400-419 149 149 149 269 269 269 F F F 21 420-439 146 146 146 264 264 264 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 148 148 148 268 268 268 F F F 24 480-499 151 151 151 271 271 271 F F F 25 500-519 151 151 151 271 271 271 T F T <--- 26 520-539 151 151 151 271 271 271 F F F 27 540-559 151 151 151 271 271 271 T F T <--- 28 560-579 148 148 148 268 268 268 T F T <--- 29 580-599 149 149 149 269 269 269 F F F Max gen: 151, Max flush: 271 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 271 271 271 F F F 1 020-039 151 151 151 271 271 271 F F F 2 040-059 151 151 151 271 271 271 F F F 3 060-079 150 150 150 270 270 270 F F F 4 080-099 150 150 150 270 270 270 F F F 5 100-119 151 151 151 271 271 271 F F F 6 120-139 152 152 152 272 272 272 F F F 7 140-159 152 152 152 272 272 272 F F F 8 160-179 152 152 152 272 272 272 F F F 9 180-199 152 152 152 272 272 272 F F F 10 200-219 152 152 152 272 272 272 F F F 11 220-239 148 148 148 267 267 267 F F F 12 240-259 147 147 147 266 266 266 F F F 13 260-279 151 151 151 271 271 271 F F F 14 280-299 152 152 152 272 272 272 F F F 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 146 146 146 263 263 263 F F F 18 360-379 138 138 138 246 246 246 F F F 19 380-399 146 146 146 265 265 265 F F F 20 400-419 149 149 149 269 269 269 F F F 21 420-439 146 146 146 264 264 264 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 148 148 148 268 268 268 F F F 24 480-499 151 151 151 271 271 271 F F F 25 500-519 152 152 152 272 272 272 F F F 26 520-539 151 151 151 271 271 271 F F F 27 540-559 152 152 152 272 272 272 F F F 28 560-579 152 152 152 272 272 272 F F F 29 580-599 149 149 149 269 269 269 F F F Max gen: 152, Max flush: 272 Begin loop 77 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.705598336Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.732557351Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.733418406Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.733503093Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.73353694Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.733580056Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811} {"msg":"Crucible dcf7acfb-2527-433f-9d3c-c18e715ce6ab has session id: 707c0e61-5098-4fad-9212-90a915618963","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.73439569Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"Upstairs opts: Upstairs UUID: dcf7acfb-2527-433f-9d3c-c18e715ce6ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.734462873Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"Crucible stats registered with UUID: dcf7acfb-2527-433f-9d3c-c18e715ce6ab","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.734492793Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:48.734518023Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.735079675Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.735149373Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.735180033Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.735867324Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.736182849Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811} {"msg":"dcf7acfb-2527-433f-9d3c-c18e715ce6ab active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.736234551Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.736446531Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"client":"0","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.736483233Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"client":"1","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.73651669Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"client":"2","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.736553272Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.736581819Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.736707718Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"ds_connection connected from Ok(127.0.0.1:49313)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.736927363Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"io task","client":"0","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"ds_connection connected from Ok(127.0.0.1:35167)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.736960088Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"io task","client":"2","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"ds_connection connected from Ok(127.0.0.1:50759)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.736989295Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"io task","client":"1","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.738666185Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"client":"2","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.738825962Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"client":"0","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.740045546Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.740121567Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"client":"1","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.740152398Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"[0]R flush_numbers[0..12]: [271, 271, 271, 270, 270, 271, 272, 272, 272, 272, 272, 267]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741139182Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"downstairs","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"[0]R generation[0..12]: [151, 151, 151, 150, 150, 151, 152, 152, 152, 152, 152, 148]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741176616Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"downstairs","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.7412089Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"downstairs","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"[1]R flush_numbers[0..12]: [271, 271, 271, 270, 270, 271, 272, 272, 272, 272, 272, 267]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741249109Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"downstairs","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"[1]R generation[0..12]: [151, 151, 151, 150, 150, 151, 152, 152, 152, 152, 152, 148]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741280081Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"downstairs","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741312004Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"downstairs","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"[2]R flush_numbers[0..12]: [271, 271, 271, 270, 270, 271, 272, 272, 272, 272, 272, 267]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741342965Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"downstairs","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"[2]R generation[0..12]: [151, 151, 151, 150, 150, 151, 152, 152, 152, 152, 152, 148]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741373395Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"downstairs","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741405158Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"downstairs","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"Max found gen is 153","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.74143113Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"downstairs","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"Generation requested: 154 >= found:153","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741458063Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"downstairs","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741483353Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"downstairs","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741508773Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"":"downstairs","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741535697Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741560125Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"client":"0","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741587098Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"client":"1","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741613721Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"client":"2","":"downstairs","session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741641386Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741670604Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"dcf7acfb-2527-433f-9d3c-c18e715ce6ab is now active with session: 707c0e61-5098-4fad-9212-90a915618963","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741699671Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741723649Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811,"session_id":"707c0e61-5098-4fad-9212-90a915618963"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.74174929Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811} {"msg":"Activated sub_volume dcf7acfb-2527-433f-9d3c-c18e715ce6ab","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:48.741771413Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 582 len 40960 data: 15 15 16 13 12 6 6 7 8 6 02/30 Write block 589 len 8192 data: 8 9 03/30 Read block 225 len 8192 04/30 Write block 155 len 40960 data: 9 9 11 11 13 14 16 15 17 15 05/30 Read block 359 len 20480 06/30 Write block 65 len 12288 data: 10 8 8 07/30 Write block 432 len 36864 data: 9 12 11 15 18 21 20 16 15 08/30 Read block 362 len 20480 09/30 Read block 82 len 12288 10/30 Write block 201 len 4096 data: 13 11/30 Write block 412 len 20480 data: 9 9 9 11 10 12/30 Write block 390 len 24576 data: 12 14 16 14 11 9 13/30 Write block 202 len 40960 data: 13 13 11 15 16 17 17 18 15 11 14/30 Read block 560 len 12288 15/30 Write block 194 len 36864 data: 16 12 14 13 14 12 15 14 14 16/30 Write block 507 len 8192 data: 8 9 17/30 Read block 158 len 8192 18/30 Read block 361 len 40960 19/30 Write block 20 len 12288 data: 13 12 13 20/30 Read block 150 len 32768 21/30 Read block 68 len 20480 22/30 Read block 417 len 24576 23/30 Write block 560 len 20480 data: 11 9 12 13 13 24/30 Write block 461 len 40960 data: 13 16 16 14 13 14 14 11 12 14 25/30 Read block 140 len 24576 26/30 Read block 374 len 4096 27/30 Write block 358 len 36864 data: 17 16 15 14 15 12 12 13 11 28/30 Read block 55 len 16384 29/30 Write block 15 len 16384 data: 9 6 7 9 30/30 Write block 214 len 4096 data: 7 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 6 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 25 2 29 Done 28 5 28 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:48.78019687Z","hostname":"bmat-EVT22200007-0000ead9","pid":1811} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 271 271 271 T F T <--- 1 020-039 151 151 151 271 271 271 T F T <--- 2 040-059 151 151 151 271 271 271 F F F 3 060-079 150 150 150 270 270 270 T F T <--- 4 080-099 150 150 150 270 270 270 F F F 5 100-119 151 151 151 271 271 271 F F F 6 120-139 152 152 152 272 272 272 F F F 7 140-159 152 152 152 272 272 272 T T T 8 160-179 152 152 152 272 272 272 T T T 9 180-199 152 152 152 272 272 272 T F T <--- 10 200-219 152 152 152 272 272 272 T F T <--- 11 220-239 148 148 148 267 267 267 F F F 12 240-259 147 147 147 266 266 266 F F F 13 260-279 151 151 151 271 271 271 F F F 14 280-299 152 152 152 272 272 272 F F F 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 146 146 146 263 263 263 T F T <--- 18 360-379 138 138 138 246 246 246 T F T <--- 19 380-399 146 146 146 265 265 265 T F T <--- 20 400-419 149 149 149 269 269 269 T F T <--- 21 420-439 146 146 146 264 264 264 T F T <--- 22 440-459 149 149 149 269 269 269 T F T <--- 23 460-479 148 148 148 268 268 268 T F T <--- 24 480-499 151 151 151 271 271 271 F F F 25 500-519 152 152 152 272 272 272 T F T <--- 26 520-539 151 151 151 271 271 271 F F F 27 540-559 152 152 152 272 272 272 F F F 28 560-579 152 152 152 272 272 272 T F T <--- 29 580-599 149 149 149 269 269 269 T T T Max gen: 152, Max flush: 272 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 273 273 273 F F F 1 020-039 153 153 153 273 273 273 F F F 2 040-059 151 151 151 271 271 271 F F F 3 060-079 153 153 153 273 273 273 F F F 4 080-099 150 150 150 270 270 270 F F F 5 100-119 151 151 151 271 271 271 F F F 6 120-139 152 152 152 272 272 272 F F F 7 140-159 153 153 153 273 273 273 F F F 8 160-179 153 153 153 273 273 273 F F F 9 180-199 153 153 153 273 273 273 F F F 10 200-219 153 153 153 273 273 273 F F F 11 220-239 148 148 148 267 267 267 F F F 12 240-259 147 147 147 266 266 266 F F F 13 260-279 151 151 151 271 271 271 F F F 14 280-299 152 152 152 272 272 272 F F F 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 153 153 153 273 273 273 F F F 18 360-379 153 153 153 273 273 273 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 153 153 153 273 273 273 F F F 21 420-439 153 153 153 273 273 273 F F F 22 440-459 153 153 153 273 273 273 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 151 151 151 271 271 271 F F F 25 500-519 153 153 153 273 273 273 F F F 26 520-539 151 151 151 271 271 271 F F F 27 540-559 152 152 152 272 272 272 F F F 28 560-579 153 153 153 273 273 273 F F F 29 580-599 153 153 153 273 273 273 F F F Max gen: 153, Max flush: 273 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.501443936Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.527534251Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.528490945Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.528926868Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.529061916Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.529343203Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819} {"msg":"Crucible d3ea580e-02bf-4d57-820d-8562b8b0f6a3 has session id: 62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.530514361Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"Upstairs opts: Upstairs UUID: d3ea580e-02bf-4d57-820d-8562b8b0f6a3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.530589681Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"Crucible stats registered with UUID: d3ea580e-02bf-4d57-820d-8562b8b0f6a3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.530630331Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:53.530773434Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.531413532Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.531500915Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.531531165Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.531784246Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.531817051Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819} {"msg":"d3ea580e-02bf-4d57-820d-8562b8b0f6a3 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.532026395Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.532060372Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"client":"0","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.532089019Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"client":"1","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.532115902Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"client":"2","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.532142876Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.532263514Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.532310047Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36136)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.532535002Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"io task","client":"2","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58740)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.532667023Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"io task","client":"1","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"ds_connection connected from Ok(127.0.0.1:43367)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.532703275Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"io task","client":"0","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.534302972Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"client":"0","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.534383572Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"client":"2","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.534583217Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"client":"1","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.534883993Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.535295046Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"[0]R flush_numbers[0..12]: [273, 273, 271, 273, 270, 271, 272, 273, 273, 273, 273, 267]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.535637895Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"downstairs","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"[0]R generation[0..12]: [153, 153, 151, 153, 150, 151, 152, 153, 153, 153, 153, 148]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.535692804Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"downstairs","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.535733224Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"downstairs","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"[1]R flush_numbers[0..12]: [273, 273, 271, 273, 270, 271, 272, 273, 273, 273, 273, 267]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.535767312Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"downstairs","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"[1]R generation[0..12]: [153, 153, 151, 153, 150, 151, 152, 153, 153, 153, 153, 148]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.535800788Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"downstairs","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.53584252Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"downstairs","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"[2]R flush_numbers[0..12]: [273, 273, 271, 273, 270, 271, 272, 273, 273, 273, 273, 267]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.535883081Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"downstairs","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"[2]R generation[0..12]: [153, 153, 151, 153, 150, 151, 152, 153, 153, 153, 153, 148]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.535917158Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"downstairs","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.535959943Z","hostname":"bmat-EVT22200007-0000ead9"Wait for a query_work_queue command to finish before sending IO ,"pid":1819,"":"downstairs","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.53606969Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"downstairs","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"Generation requested: 156 >= found:154","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.53611019Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"downstairs","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.536139228Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"downstairs","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.536167313Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"":"downstairs","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.536196982Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.536223334Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"client":"0","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.536253053Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"client":"1","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.536282261Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"client":"2","":"downstairs","session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.536312301Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.536340576Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"d3ea580e-02bf-4d57-820d-8562b8b0f6a3 is now active with session: 62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.536372389Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819,"session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.536399122Z","hostname":"bmat-EVT22200007-0000ead9","pid":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 1819,"session_id":"62b4ab8e-ce80-4c52-bc5c-bf7caaf2527b"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.536436126Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819} {"msg":"Activated sub_volume d3ea580e-02bf-4d57-820d-8562b8b0f6a3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:53.536460644Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 594 len 12288 data: 8 9 10 02/30 Read block 232 len 36864 03/30 Flush 04/30 Flush 05/30 Write block 418 len 8192 data: 6 4 06/30 Write block 69 len 28672 data: 13 13 11 12 12 10 10 07/30 Write block 419 len 12288 data: 5 6 5 08/30 Write block 292 len 16384 data: 15 10 12 13 09/30 Write block 537 len 12288 data: 15 15 15 10/30 Read block 405 len 16384 11/30 Read block 105 len 8192 12/30 Read block 415 len 28672 13/30 Flush 14/30 Write block 228 len 20480 data: 11 10 11 12 12 15/30 Read block 225 len 12288 16/30 Read block 403 len 40960 17/30 Read block 374 len 36864 18/30 Read block 374 len 20480 19/30 Read block 521 len 20480 20/30 Read block 254 len 12288 21/30 Read block 96 len 20480 22/30 Read block 443 len 40960 23/30 Write block 345 len 32768 data: 10 11 10 9 10 10 8 7 24/30 Read block 427 len 36864 25/30 Read block 419 len 8192 26/30 Flush 27/30 Write block 526 len 32768 data: 7 8 10 12 11 12 13 16 28/30 Write block 589 len 36864 data: 9 10 7 9 8 9 10 11 10 29/30 Read block 429 len 36864 30/30 Write block 105 len 28672 data: 7 6 6 9 8 11 12 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:53.573269491Z","hostname":"bmat-EVT22200007-0000ead9","pid":1819} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 273 273 273 F F F 1 020-039 153 153 153 273 273 273 F F F 2 040-059 151 151 151 271 271 271 F F F 3 060-079 156 153 156 276 273 276 F F F <--- 4 080-099 150 150 150 270 270 270 F F F 5 100-119 151 151 151 271 271 271 T F T <--- 6 120-139 152 152 152 272 272 272 F F F 7 140-159 153 153 153 273 273 273 F F F 8 160-179 153 153 153 273 273 273 F F F 9 180-199 153 153 153 273 273 273 F F F 10 200-219 153 153 153 273 273 273 F F F 11 220-239 156 148 156 277 267 277 F F F <--- 12 240-259 147 147 147 266 266 266 F F F 13 260-279 151 151 151 271 271 271 F F F 14 280-299 156 152 156 276 272 276 F F F <--- 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 156 153 156 277 273 277 F F F <--- 18 360-379 153 153 153 273 273 273 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 156 153 156 276 273 276 F F F <--- 21 420-439 156 153 156 276 273 276 F F F <--- 22 440-459 153 153 153 273 273 273 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 151 151 151 271 271 271 F F F 25 500-519 153 153 153 273 273 273 F F F 26 520-539 156 151 156 276 271 276 T F T <--- 27 540-559 152 152 152 272 272 272 F F F 28 560-579 153 153 153 273 273 273 F F F 29 580-599 156 153 156 274 273 274 T T T <--- Max gen: 156, Max flush: 277 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 273 273 273 F F F 1 020-039 153 153 153 273 273 273 F F F 2 040-059 151 151 151 271 271 271 F F F 3 060-079 156 156 156 276 276 276 F F F 4 080-099 150 150 150 270 270 270 F F F 5 100-119 157 157 157 278 278 278 F F F 6 120-139 152 152 152 272 272 272 F F F 7 140-159 153 153 153 273 273 273 F F F 8 160-179 153 153 153 273 273 273 F F F 9 180-199 153 153 153 273 273 273 F F F 10 200-219 153 153 153 273 273 273 F F F 11 220-239 156 156 156 277 277 277 F F F 12 240-259 147 147 147 266 266 266 F F F 13 260-279 151 151 151 271 271 271 F F F 14 280-299 156 156 156 276 276 276 F F F 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 156 156 156 277 277 277 F F F 18 360-379 153 153 153 273 273 273 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 156 156 156 276 276 276 F F F 21 420-439 156 156 156 276 276 276 F F F 22 440-459 153 153 153 273 273 273 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 151 151 151 271 271 271 F F F 25 500-519 153 153 153 273 273 273 F F F 26 520-539 157 157 157 278 278 278 F F F 27 540-559 152 152 152 272 272 272 F F F 28 560-579 153 153 153 273 273 273 F F F 29 580-599 157 157 157 278 278 278 F F F Max gen: 157, Max flush: 278 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.204952119Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:58.230466023Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.230559228Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.255578062Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.256116438Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.256252587Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.256298258Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.256355221Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827} {"msg":"Crucible b552f64a-cdef-4d9d-8bcc-3ae1e4db3dec has session id: 1d2b412f-8051-4ec6-819e-95d1325d9efe","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.257492452Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"Upstairs opts: Upstairs UUID: b552f64a-cdef-4d9d-8bcc-3ae1e4db3dec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.257587801Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"Crucible stats registered with UUID: b552f64a-cdef-4d9d-8bcc-3ae1e4db3dec","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.257641908Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:58.257687057Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.25813306Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.258222177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.258252126Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.258499145Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.258532571Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827} {"msg":"b552f64a-cdef-4d9d-8bcc-3ae1e4db3dec active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.258700143Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.258731375Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"client":"0","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.258758559Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"client":"1","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.258785342Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"client":"2","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.259120075Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.259217237Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.259247517Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"ds_connection connected from Ok(127.0.0.1:49729)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.259277166Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"io task","client":"2","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"ds_connection connected from Ok(127.0.0.1:34740)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.259306273Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"io task","client":"1","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"ds_connection connected from Ok(127.0.0.1:41747)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.259335241Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"io task","client":"0","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.261232056Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"client":"1","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.261312606Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"client":"2","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.261914738Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.262368596Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.262582089Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"client":"0","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"[0]R flush_numbers[0..12]: [273, 273, 271, 276, 270, 278, 272, 273, 273, 273, 273, 277]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.263373215Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"downstairs","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"[0]R generation[0..12]: [153, 153, 151, 156, 150, 157, 152, 153, 153, 153, 153, 156]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.263494465Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"downstairs","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.2635473Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"downstairs","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"[1]R flush_numbers[0..12]: [273, 273, 271, 276, 270, 278, 272, 273, 273, 273, 273, 277]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.263598611Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"downstairs","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"[1]R generation[0..12]: [153, 153, 151, 156, 150, 157, 152, 153, 153, 153, 153, 156]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.263649863Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"downstairs","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.263703148Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"downstairs","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"[2]R flush_numbers[0..12]: [273, 273, 271, 276, 270, 278, 272, 273, 273, 273, 273, 277]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.263752436Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"downstairs","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"[2]R generation[0..12]: [153, 153, 151, 156, 150, 157, 152, 153, 153, 153, 153, 156]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.263800451Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"downstairs","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.263846783Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"downstairs","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.263920118Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.263985598Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"downstairs","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.264063833Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"downstairs","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.26410287Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"":"downstairs","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.264145685Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.264185213Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"client":"0","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.264236494Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"client":"1","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.26427956Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"client":"2","":"downstairs","session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.264322394Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.264363235Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"b552f64a-cdef-4d9d-8bcc-3ae1e4db3dec is now active with session: 1d2b412f-8051-4ec6-819e-95d1325d9efe","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.264409908Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.264447602Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827,"session_id":"1d2b412f-8051-4ec6-819e-95d1325d9efe"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.264483874Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Activated sub_volume b552f64a-cdef-4d9d-8bcc-3ae1e4db3dec","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:15:58.264522771Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 518 len 16384 data: 8 9 7 7 02/30 Read block 61 len 32768 03/30 Write block 116 len 16384 data: 17 19 17 14 04/30 Write block 199 len 40960 data: 13 16 15 15 14 12 16 17 18 18 05/30 Read block 266 len 28672 06/30 Read block 352 len 20480 07/30 Read block 20 len 32768 08/30 Write block 199 len 40960 data: 14 17 16 16 15 13 17 18 19 19 09/30 Read block 351 len 32768 10/30 Write block 416 len 8192 data: 11 7 11/30 Write block 406 len 16384 data: 15 14 14 11 12/30 Read block 517 len 20480 13/30 Read block 37 len 24576 14/30 Read block 558 len 16384 15/30 Write block 362 len 16384 data: 16 13 13 14 16/30 Read block 88 len 24576 17/30 Write block 540 len 8192 data: 13 13 18/30 Read block 160 len 28672 19/30 Write block 186 len 4096 data: 12 20/30 Read block 484 len 40960 21/30 Write block 563 len 20480 data: 14 14 11 11 13 22/30 Read block 74 len 36864 23/30 Write block 229 len 32768 data: 11 12 13 13 14 16 15 17 24/30 Write block 17 len 28672 data: 8 10 12 14 13 14 10 25/30 Read block 385 len 4096 26/30 Read block 303 len 36864 27/30 Write block 181 len 32768 data: 13 13 11 12 12 13 13 14 28/30 Read block 83 len 36864 29/30 Read block 273 len 24576 30/30 Write block 274 len 40960 data: 15 14 14 15 15 14 14 13 13 15 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Write 4 Sent Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Read 8 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:15:58.305849242Z","hostname":"bmat-EVT22200007-0000ead9","pid":1827} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 273 273 273 F T T <--- 1 020-039 153 153 153 273 273 273 F T T <--- 2 040-059 151 151 151 271 271 271 F F F 3 060-079 156 156 156 276 276 276 F F F 4 080-099 150 150 150 270 270 270 F F F 5 100-119 157 157 157 278 278 278 F T T <--- 6 120-139 152 152 152 272 272 272 F F F 7 140-159 153 153 153 273 273 273 F F F 8 160-179 153 153 153 273 273 273 F F F 9 180-199 153 153 153 273 273 273 F T T <--- 10 200-219 153 153 153 273 273 273 F T T <--- 11 220-239 156 156 156 277 277 277 F T T <--- 12 240-259 147 147 147 266 266 266 F F F 13 260-279 151 151 151 271 271 271 F T T <--- 14 280-299 156 156 156 276 276 276 F T T <--- 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 156 156 156 277 277 277 F F F 18 360-379 153 153 153 273 273 273 F T T <--- 19 380-399 153 153 153 273 273 273 F F F 20 400-419 156 156 156 276 276 276 F T T <--- 21 420-439 156 156 156 276 276 276 F F F 22 440-459 153 153 153 273 273 273 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 151 151 151 271 271 271 F F F 25 500-519 153 153 153 273 273 273 T T T 26 520-539 157 157 157 278 278 278 T T T 27 540-559 152 152 152 272 272 272 F T T <--- 28 560-579 153 153 153 273 273 273 F T T <--- 29 580-599 157 157 157 278 278 278 F F F Max gen: 157, Max flush: 278 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 279 279 279 F F F 1 020-039 158 158 158 279 279 279 F F F 2 040-059 151 151 151 271 271 271 F F F 3 060-079 156 156 156 276 276 276 F F F 4 080-099 150 150 150 270 270 270 F F F 5 100-119 158 158 158 279 279 279 F F F 6 120-139 152 152 152 272 272 272 F F F 7 140-159 153 153 153 273 273 273 F F F 8 160-179 153 153 153 273 273 273 F F F 9 180-199 158 158 158 279 279 279 F F F 10 200-219 158 158 158 279 279 279 F F F 11 220-239 158 158 158 279 279 279 F F F 12 240-259 147 147 147 266 266 266 F F F 13 260-279 158 158 158 279 279 279 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 152 152 152 272 272 272 F F F 16 320-339 152 152 152 272 272 272 F F F 17 340-359 156 156 156 277 277 277 F F F 18 360-379 158 158 158 279 279 279 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 158 158 158 279 279 279 F F F 21 420-439 156 156 156 276 276 276 F F F 22 440-459 153 153 153 273 273 273 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 151 151 151 271 271 271 F F F 25 500-519 158 158 158 279 279 279 F F F 26 520-539 158 158 158 279 279 279 F F F 27 540-559 158 158 158 279 279 279 F F F 28 560-579 158 158 158 279 279 279 F F F 29 580-599 157 157 157 278 278 278 F F F Max gen: 158, Max flush: 279 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.011040879Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.037625142Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.038212765Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.038317302Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.038345578Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.038389215Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835} {"msg":"Crucible e5011262-631d-43ff-b33e-9c2d16a453ef has session id: 033d247a-c1a8-453b-a688-9f04ae746b8c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.039537368Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"Upstairs opts: Upstairs UUID: e5011262-631d-43ff-b33e-9c2d16a453ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.039612476Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"Crucible stats registered with UUID: e5011262-631d-43ff-b33e-9c2d16a453ef","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.039648097Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:03.039678818Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.040151464Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.040213146Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.040247744Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.04074064Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.040779647Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835} {"msg":"e5011262-631d-43ff-b33e-9c2d16a453ef active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.041024131Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.041060594Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"client":"0","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.041093779Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"client":"1","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.041126404Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"client":"2","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.041307833Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.041357812Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.041387461Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43698)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.041702504Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"io task","client":"2","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59113)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.041768756Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"io task","client":"1","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:65427)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.041801691Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"io task","client":"0","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.043697775Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"client":"2","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.044181212Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"client":"0","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.044234838Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"client":"1","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.045049671Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.045149198Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"[0]R flush_numbers[0..12]: [279, 279, 271, 276, 270, 279, 272, 273, 273, 279, 279, 279]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.045548128Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"downstairs","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"[0]R generation[0..12]: [158, 158, 151, 156, 150, 158, 152, 153, 153, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.045607676Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"downstairs","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.045668697Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"downstairs","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"[1]R flush_numbers[0..12]: [279, 279, 271, 276, 270, 279, 272, 273, 273, 279, 279, 279]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.045719337Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"downstairs","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"[1]R generation[0..12]: [158, 158, 151, 156, 150, 158, 152, 153, 153, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.045763505Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"downstairs","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.045807852Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"downstairs","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"[2]R flush_numbers[0..12]: [279, 279, 271, 276, 270, 279, 272, 273, 273, 279, 279, 279]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.045936758Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"downstairs","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"[2]R generation[0..12]: [158, 158, 151, 156, 150, 158, 152, 153, 153, 158, 158, 158]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:16:03.046034291Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"downstairs","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.046179819Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"downstairs","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"Max found gen is 159","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.04632692Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"downstairs","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"Generation requested: 160 >= found:159","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.046522447Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"downstairs","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.046561234Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"downstairs","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.04658937Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"":"downstairs","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.046628277Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.046653627Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"client":"0","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.046680791Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"client":"1","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.046707965Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"client":"2","":"downstairs","session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.04673621Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.046772623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"e5011262-631d-43ff-b33e-9c2d16a453ef is now active with session: 033d247a-c1a8-453b-a688-9f04ae746b8c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.046810648Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.046835537Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835,"session_id":"033d247a-c1a8-453b-a688-9f04ae746b8c"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.046861709Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835} {"msg":"Activated sub_volume e5011262-631d-43ff-b33e-9c2d16a453ef","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:03.046884324Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 257 len 16384 data: 12 12 13 16 02/30 Flush 03/30 Write block 375 len 12288 data: 7 8 10 04/30 Read block 491 len 28672 05/30 Read block 482 len 28672 06/30 Flush 07/30 Write block 410 len 4096 data: 11 08/30 Write block 309 len 4096 data: 15 09/30 Write block 484 len 12288 data: 9 9 10 10/30 Write block 342 len 4096 data: 15 11/30 Read block 65 len 4096 12/30 Write block 103 len 36864 data: 4 6 8 7 7 10 9 12 13 13/30 Write block 43 len 12288 data: 12 12 15 14/30 Write block 148 len 28672 data: 13 9 9 12 9 10 11 15/30 Flush 16/30 Write block 585 len 8192 data: 14 13 17/30 Read block 394 len 32768 18/30 Write block 324 len 16384 data: 14 9 10 8 19/30 Read block 196 len 40960 20/30 Read block 529 len 4096 21/30 Write block 152 len 28672 data: 10 11 12 10 10 12 12 22/30 Flush 23/30 Write block 261 len 8192 data: 17 18 24/30 Write block 511 len 24576 data: 10 11 10 12 11 9 25/30 Read block 355 len 36864 26/30 Write block 376 len 4096 data: 9 27/30 Write block 94 len 8192 data: 14 14 28/30 Read block 565 len 28672 29/30 Read block 3 len 8192 30/30 Read block 474 len 20480 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 1 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Write 7 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 24 0 25 Done 23 0 24 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1005)) Some(JobId(1021)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:03.07817776Z","hostname":"bmat-EVT22200007-0000ead9","pid":1835} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 279 279 279 F F F 1 020-039 158 158 158 279 279 279 F F F 2 040-059 160 151 160 282 271 282 F F F <--- 3 060-079 156 156 156 276 276 276 F F F 4 080-099 150 150 150 270 270 270 T F T <--- 5 100-119 160 158 160 282 279 282 F F F <--- 6 120-139 152 152 152 272 272 272 F F F 7 140-159 160 153 160 283 273 283 F F F <--- 8 160-179 153 153 153 273 273 273 F F F 9 180-199 158 158 158 279 279 279 F F F 10 200-219 158 158 158 279 279 279 F F F 11 220-239 158 158 158 279 279 279 F F F 12 240-259 160 160 160 280 280 280 F F F 13 260-279 160 160 160 280 280 280 T F T <--- 14 280-299 158 158 158 279 279 279 F F F 15 300-319 160 152 160 282 272 282 F F F <--- 16 320-339 160 152 160 283 272 283 F F F <--- 17 340-359 160 156 160 282 277 282 F F F <--- 18 360-379 160 160 160 281 281 281 T F T <--- 19 380-399 153 153 153 273 273 273 F F F 20 400-419 160 158 160 282 279 282 F F F <--- 21 420-439 156 156 156 276 276 276 F F F 22 440-459 153 153 153 273 273 273 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 160 151 160 282 271 282 F F F <--- 25 500-519 158 158 158 279 279 279 T F T <--- 26 520-539 158 158 158 279 279 279 F F F 27 540-559 158 158 158 279 279 279 F F F 28 560-579 158 158 158 279 279 279 F F F 29 580-599 160 157 160 283 278 283 F F F <--- Max gen: 160, Max flush: 283 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 279 279 279 F F F 1 020-039 158 158 158 279 279 279 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 156 156 156 276 276 276 F F F 4 080-099 161 161 161 284 284 284 F F F 5 100-119 160 160 160 282 282 282 F F F 6 120-139 152 152 152 272 272 272 F F F 7 140-159 160 160 160 283 283 283 F F F 8 160-179 153 153 153 273 273 273 F F F 9 180-199 158 158 158 279 279 279 F F F 10 200-219 158 158 158 279 279 279 F F F 11 220-239 158 158 158 279 279 279 F F F 12 240-259 160 160 160 280 280 280 F F F 13 260-279 161 161 161 284 284 284 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 160 160 160 282 282 282 F F F 16 320-339 160 160 160 283 283 283 F F F 17 340-359 160 160 160 282 282 282 F F F 18 360-379 161 161 161 284 284 284 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 160 160 160 282 282 282 F F F 21 420-439 156 156 156 276 276 276 F F F 22 440-459 153 153 153 273 273 273 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 160 160 160 282 282 282 F F F 25 500-519 161 161 161 284 284 284 F F F 26 520-539 158 158 158 279 279 279 F F F 27 540-559 158 158 158 279 279 279 F F F 28 560-579 158 158 158 279 279 279 F F F 29 580-599 160 160 160 283 283 283 F F F Max gen: 161, Max flush: 284 Begin loop 81 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.735211351Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.770554526Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.771195795Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.771293168Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.771321013Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.771360021Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843} {"msg":"Crucible e4aea90c-8a58-4268-bd53-c3b772f204ac has session id: 8f4d98e5-0331-4978-adc3-0d2e03b8ff71","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.77250101Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Upstairs opts: Upstairs UUID: e4aea90c-8a58-4268-bd53-c3b772f204ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.77296079Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Crucible stats registered with UUID: e4aea90c-8a58-4268-bd53-c3b772f204ac","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.773005108Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:07.773033544Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.77361755Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.7736832Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.773715073Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.77412772Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.774173802Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843} {"msg":"e4aea90c-8a58-4268-bd53-c3b772f204ac active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.774288048Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.774318438Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"client":"0","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.774355541Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"client":"1","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.77438523Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"client":"2","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.774414358Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.774444608Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.774481962Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"ds_connection connected from Ok(127.0.0.1:33218)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.775026018Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"io task","client":"2","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"ds_connection connected from Ok(127.0.0.1:33620)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.775100105Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"io task","client":"1","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"ds_connection connected from Ok(127.0.0.1:45033)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.775133821Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"io task","client":"0","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.776603391Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"client":"0","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.776649102Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"client":"1","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.777189521Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"client":"2","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.777665434Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.777811393Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"[0]R flush_numbers[0..12]: [279, 279, 282, 276, 284, 282, 272, 283, 273, 279, 279, 279]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778150364Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"downstairs","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"[0]R generation[0..12]: [158, 158, 160, 156, 161, 160, 152, 160, 153, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778191515Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"downstairs","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778233989Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"downstairs","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"[1]R flush_numbers[0..12]: [279, 279, 282, 276, 284, 282, 272, 283, 273, 279, 279, 279]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778268207Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"downstairs","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"[1]R generation[0..12]: [158, 158, 160, 156, 161, 160, 152, 160, 153, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778301804Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"downstairs","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778336522Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"downstairs","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"[2]R flush_numbers[0..12]: [279, 279, 282, 276, 284, 282, 272, 283, 273, 279, 279, 279]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778370329Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"downstairs","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"[2]R generation[0..12]: [158, 158, 160, 156, 161, 160, 152, 160, 153, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778403725Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"downstairs","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778438955Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"downstairs","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778467672Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"downstairs","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778497671Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"downstairs","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Next flush: 285","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778525517Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"downstairs","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778553833Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"":"downstairs","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778598481Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778626537Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"client":"0","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778656025Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"client":"1","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.77868906Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"client":"2","":"downstairs","session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778719841Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778748097Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"e4aea90c-8a58-4268-bd53-c3b772f204ac is now active with session: 8f4d98e5-0331-4978-adc3-0d2e03b8ff71","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.77877996Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778806523Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843,"session_id":"8f4d98e5-0331-4978-adc3-0d2e03b8ff71"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778836913Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843} {"msg":"Activated sub_volume e4aea90c-8a58-4268-bd53-c3b772f204ac","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:07.778861121Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 307 len 36864 data: 15 15 16 15 14 10 8 6 8 02/30 Write block 445 len 12288 data: 14 16 16 03/30 Read block 181 len 20480 04/30 Flush 05/30 Write block 75 len 8192 data: 11 8 06/30 Write block 368 len 20480 data: 12 10 11 11 11 07/30 Write block 209 len 36864 data: 19 16 12 12 9 8 7 10 12 08/30 Flush 09/30 Write block 116 len 4096 data: 18 10/30 Read block 401 len 20480 11/30 Read block 406 len 28672 12/30 Read block 43 len 4096 13/30 Read block 355 len 32768 14/30 Write block 312 len 40960 data: 11 9 7 9 11 9 9 10 15 13 15/30 Write block 435 len 4096 data: 16 16/30 Read block 555 len 16384 17/30 Write block 158 len 32768 data: 13 14 15 17 16 18 16 15 18/30 Read block 361 len 24576 19/30 Read block 516 len 8192 20/30 Read block 534 len 8192 21/30 Flush 22/30 Write block 561 len 32768 data: 10 13 15 15 12 12 14 12 23/30 Read block 159 len 24576 24/30 Read block 86 len 36864 25/30 Read block 153 len 32768 26/30 Write block 83 len 32768 data: 10 13 13 14 14 12 12 9 27/30 Read block 472 len 20480 28/30 Write block 231 len 24576 data: 14 14 15 17 16 18 29/30 Read block 105 len 4096 30/30 Read block 4 len 28672 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 27 28 Done 30 29 3 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:07.819450853Z","hostname":"bmat-EVT22200007-0000ead9","pid":1843} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 279 279 279 F F F 1 020-039 158 158 158 279 279 279 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 162 162 156 286 286 276 F F F <--- 4 080-099 161 161 161 284 284 284 T T F <--- 5 100-119 162 162 160 287 287 282 F F F <--- 6 120-139 152 152 152 272 272 272 F F F 7 140-159 162 162 160 287 287 283 F F F <--- 8 160-179 162 162 153 287 287 273 F F F <--- 9 180-199 158 158 158 279 279 279 F F F 10 200-219 162 162 158 286 286 279 F F F <--- 11 220-239 158 158 158 279 279 279 T T F <--- 12 240-259 160 160 160 280 280 280 F F F 13 260-279 161 161 161 284 284 284 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 162 162 160 287 287 282 F F T <--- 16 320-339 162 162 160 287 287 283 F F F <--- 17 340-359 160 160 160 282 282 282 F F F 18 360-379 162 162 161 286 286 284 F F F <--- 19 380-399 153 153 153 273 273 273 F F F 20 400-419 160 160 160 282 282 282 F F F 21 420-439 162 162 156 287 287 276 F F F <--- 22 440-459 162 162 153 285 285 273 F F T <--- 23 460-479 153 153 153 273 273 273 F F F 24 480-499 160 160 160 282 282 282 F F F 25 500-519 161 161 161 284 284 284 F F F 26 520-539 158 158 158 279 279 279 F F F 27 540-559 158 158 158 279 279 279 F F F 28 560-579 158 158 158 279 279 279 T T F <--- 29 580-599 160 160 160 283 283 283 F F F Max gen: 162, Max flush: 287 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 279 279 279 F F F 1 020-039 158 158 158 279 279 279 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 162 162 162 286 286 286 F F F 4 080-099 163 163 163 288 288 288 F F F 5 100-119 162 162 162 287 287 287 F F F 6 120-139 152 152 152 272 272 272 F F F 7 140-159 162 162 162 287 287 287 F F F 8 160-179 162 162 162 287 287 287 F F F 9 180-199 158 158 158 279 279 279 F F F 10 200-219 162 162 162 286 286 286 F F F 11 220-239 163 163 163 288 288 288 F F F 12 240-259 160 160 160 280 280 280 F F F 13 260-279 161 161 161 284 284 284 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 162 162 162 287 287 287 F F F 16 320-339 162 162 162 287 287 287 F F F 17 340-359 160 160 160 282 282 282 F F F 18 360-379 162 162 162 286 286 286 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 160 160 160 282 282 282 F F F 21 420-439 162 162 162 287 287 287 F F F 22 440-459 162 162 162 285 285 285 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 160 160 160 282 282 282 F F F 25 500-519 161 161 161 284 284 284 F F F 26 520-539 158 158 158 279 279 279 F F F 27 540-559 158 158 158 279 279 279 F F F 28 560-579 163 163 163 288 288 288 F F F 29 580-599 160 160 160 283 283 283 F F F Max gen: 163, Max flush: 288 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.586109376Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:12.609794099Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.609901201Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.634661974Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.635307262Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.635457178Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.635484843Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.635520895Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851} {"msg":"Crucible 7937bb18-006b-439d-bffb-db42b7014faa has session id: 79a06ff6-94c8-409c-8b0b-929a902ad1af","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.636525674Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"Upstairs opts: Upstairs UUID: 7937bb18-006b-439d-bffb-db42b7014faa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.636596704Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"Crucible stats registered with UUID: 7937bb18-006b-439d-bffb-db42b7014faa","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.636627776Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:12.636653216Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.637014882Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.637078829Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.637107826Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.637258705Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.637289496Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851} {"msg":"7937bb18-006b-439d-bffb-db42b7014faa active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.637385055Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.637413321Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"client":"0","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.637441697Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"client":"1","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.637678637Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"client":"2","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.637767913Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.638122034Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.638194899Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"ds_connection connected from Ok(127.0.0.1:58445)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.638234818Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"io task","client":"2","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"ds_connection connected from Ok(127.0.0.1:61408)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.638279667Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"io task","client":"1","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"ds_connection connected from Ok(127.0.0.1:58478)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.638318734Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"io task","client":"0","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.640129469Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"client":"2","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.640208936Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"client":"1","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.640298694Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"client":"0","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.641462598Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.641644829Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"[0]R flush_numbers[0..12]: [279, 279, 282, 286, 288, 287, 272, 287, 287, 279, 286, 288]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.641851208Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"downstairs","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"[0]R generation[0..12]: [158, 158, 160, 162, 163, 162, 152, 162, 162, 158, 162, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.641901597Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"downstairs","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.64194346Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"downstairs","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"[1]R flush_numbers[0..12]: [279, 279, 282, 286, 288, 287, 272, 287, 287, 279, 286, 288]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642000864Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"downstairs","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"[1]R generation[0..12]: [158, 158, 160, 162, 163, 162, 152, 162, 162, 158, 162, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642031164Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"downstairs","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642063047Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"downstairs","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"[2]R flush_numbers[0..12]: [279, 279, 282, 286, 288, 287, 272, 287, 287, 279, 286, 288]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642093598Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"downstairs","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"[2]R generation[0..12]: [158, 158, 160, 162, 163, 162, 152, 162, 162, 158, 162, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642130431Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"downstairs","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642162594Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"downstairs","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642188355Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"downstairs","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642215389Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"downstairs","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642240889Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"downstairs","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.64226647Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"":"downstairs","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642293423Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"session_id":"Wait for a query_work_queue command to finish before sending IO 79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642333292Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"client":"0","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642360346Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"client":"1","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642387239Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"client":"2","":"downstairs","session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642414543Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642440144Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"7937bb18-006b-439d-bffb-db42b7014faa is now active with session: 79a06ff6-94c8-409c-8b0b-929a902ad1af","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642468951Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642493189Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851,"session_id":"79a06ff6-94c8-409c-8b0b-929a902ad1af"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642518729Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851} {"msg":"Activated sub_volume 7937bb18-006b-439d-bffb-db42b7014faa","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:12.642540753Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 139 len 16384 data: 18 14 11 9 02/30 Write block 482 len 24576 data: 12 11 10 10 11 11 03/30 Write block 126 len 12288 data: 13 14 13 04/30 Flush 05/30 Write block 34 len 12288 data: 9 8 11 06/30 Flush 07/30 Write block 487 len 20480 data: 12 11 9 10 9 08/30 Write block 17 len 28672 data: 9 11 13 15 14 15 11 09/30 Read block 274 len 4096 10/30 Flush 11/30 Write block 521 len 12288 data: 8 10 10 12/30 Write block 231 len 28672 data: 15 15 16 18 17 19 14 13/30 Read block 215 len 32768 14/30 Flush 15/30 Write block 584 len 32768 data: 17 15 14 7 7 10 11 8 16/30 Read block 491 len 40960 17/30 Write block 302 len 32768 data: 11 13 14 14 14 16 16 17 18/30 Read block 586 len 36864 19/30 Write block 357 len 12288 data: 13 18 17 20/30 Write block 313 len 16384 data: 10 8 10 12 21/30 Read block 269 len 24576 22/30 Write block 231 len 40960 data: 16 16 17 19 18 20 15 16 18 17 23/30 Write block 116 len 40960 data: 19 20 18 15 15 11 9 8 9 9 24/30 Write block 400 len 32768 data: 14 16 14 14 16 15 16 15 25/30 Read block 466 len 20480 26/30 Write block 229 len 4096 data: 12 27/30 Write block 66 len 16384 data: 9 9 12 14 28/30 Read block 311 len 4096 29/30 Read block 510 len 40960 30/30 Read block 525 len 36864 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 6 Sent Done Done false 1002 Acked Write 3 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 3 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Read 1 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Write 7 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:12.678069345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1851} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 164 164 279 291 291 F F F <--- 1 020-039 158 164 164 279 291 291 F F F <--- 2 040-059 160 160 160 282 282 282 F F F 3 060-079 162 162 162 286 286 286 F T T <--- 4 080-099 163 163 163 288 288 288 F F F 5 100-119 162 162 162 287 287 287 F T T <--- 6 120-139 152 164 164 272 289 289 T T T <--- 7 140-159 162 164 164 287 289 289 T F F <--- 8 160-179 162 162 162 287 287 287 F F F 9 180-199 158 158 158 279 279 279 F F F 10 200-219 162 162 162 286 286 286 F F F 11 220-239 163 164 164 288 292 292 F T T <--- 12 240-259 160 160 160 280 280 280 F T T <--- 13 260-279 161 161 161 284 284 284 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 162 162 162 287 287 287 F T T <--- 16 320-339 162 162 162 287 287 287 F F F 17 340-359 160 160 160 282 282 282 F T T <--- 18 360-379 162 162 162 286 286 286 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 160 160 160 282 282 282 F T T <--- 21 420-439 162 162 162 287 287 287 F F F 22 440-459 162 162 162 285 285 285 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 160 164 164 282 291 291 F F F <--- 25 500-519 161 161 161 284 284 284 F F F 26 520-539 158 164 164 279 292 292 F F F <--- 27 540-559 158 158 158 279 279 279 F F F 28 560-579 163 163 163 288 288 288 F F F 29 580-599 160 160 160 283 283 283 F T T <--- Max gen: 164, Max flush: 292 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 291 291 291 F F F 1 020-039 164 164 164 291 291 291 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 165 165 165 293 293 293 F F F 4 080-099 163 163 163 288 288 288 F F F 5 100-119 165 165 165 293 293 293 F F F 6 120-139 165 165 165 293 293 293 F F F 7 140-159 164 164 164 289 289 289 F F F 8 160-179 162 162 162 287 287 287 F F F 9 180-199 158 158 158 279 279 279 F F F 10 200-219 162 162 162 286 286 286 F F F 11 220-239 165 165 165 293 293 293 F F F 12 240-259 165 165 165 293 293 293 F F F 13 260-279 161 161 161 284 284 284 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 165 165 165 293 293 293 F F F 16 320-339 162 162 162 287 287 287 F F F 17 340-359 165 165 165 293 293 293 F F F 18 360-379 162 162 162 286 286 286 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 165 165 165 293 293 293 F F F 21 420-439 162 162 162 287 287 287 F F F 22 440-459 162 162 162 285 285 285 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 164 164 164 291 291 291 F F F 25 500-519 161 161 161 284 284 284 F F F 26 520-539 164 164 164 292 292 292 F F F 27 540-559 158 158 158 279 279 279 F F F 28 560-579 163 163 163 288 288 288 F F F 29 580-599 165 165 165 293 293 293 F F F Max gen: 165, Max flush: 293 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.38208842Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.409864493Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.410574539Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.410721109Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.410757111Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.41081753Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859} {"msg":"Crucible ced4c897-2a31-4fb2-9740-cba3831f639a has session id: a6ff7009-4da2-4f5f-b346-cddc3794cc3e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.412170599Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"Upstairs opts: Upstairs UUID: ced4c897-2a31-4fb2-9740-cba3831f639a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.412259846Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"Crucible stats registered with UUID: ced4c897-2a31-4fb2-9740-cba3831f639a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.412297841Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:17.412329273Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.412933459Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.413027606Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.413069529Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.413620209Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.413677793Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859} {"msg":"ced4c897-2a31-4fb2-9740-cba3831f639a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.41385291Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.413884382Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"client":"0","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.413912177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"client":"1","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.413947507Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"client":"2","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.414058286Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.414093626Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.414121041Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61094)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.414562805Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"1","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:17.414632082Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:17.414669166Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:17.414704325Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64309)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.414729786Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"0","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.414757871Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.414784965Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.41687835Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"client":"0","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.417051132Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"client":"1","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.417756328Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:17.417977847Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.41703282Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43963)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.417151345Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"io task","client":"2","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.42001754Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"client":"2","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"[0]R flush_numbers[0..12]: [291, 291, 282, 293, 288, 293, 293, 289, 287, 279, 286, 293]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421165412Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"downstairs","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"[0]R generation[0..12]: [164, 164, 160, 165, 163, 165, 165, 164, 162, 158, 162, 165]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.42126003Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"downstairs","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421297674Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"downstairs","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"[1]R flush_numbers[0..12]: [291, 291, 282, 293, 288, 293, 293, 289, 287, 279, 286, 293]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421332423Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"downstairs","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"[1]R generation[0..12]: [164, 164, 160, 165, 163, 165, 165, 164, 162, 158, 162, 165]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421365719Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"downstairs","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421400868Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"downstairs","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"[2]R flush_numbers[0..12]: [291, 291, 282, 293, 288, 293, 293, 289, 287, 279, 286, 293]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421434665Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"downstairs","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"[2]R generation[0..12]: [164, 164, 160, 165, 163, 165, 165, 164, 162, 158, 162, 165]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:16:27.421475496Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"downstairs","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421523802Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"downstairs","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421552859Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"downstairs","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421583099Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"downstairs","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421611445Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"downstairs","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421639481Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"":"downstairs","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421668939Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 bmat-EVT22200007-0000ead9","pid":1859,"session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421703728Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"client":"0","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421734098Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"client":"1","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421763647Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"client":"2","":"downstairs","session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421794177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421822483Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"ced4c897-2a31-4fb2-9740-cba3831f639a is now active with session: a6ff7009-4da2-4f5f-b346-cddc3794cc3e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421854326Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421881039Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859,"session_id":"a6ff7009-4da2-4f5f-b346-cddc3794cc3e"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421909335Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859} {"msg":"Activated sub_volume ced4c897-2a31-4fb2-9740-cba3831f639a","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:27.421933523Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 361 len 16384 data: 15 17 14 14 02/30 Read block 563 len 36864 03/30 Write block 3 len 40960 data: 11 13 15 17 17 11 11 10 12 11 04/30 Read block 591 len 36864 05/30 Write block 8 len 40960 data: 12 12 11 13 12 10 9 10 7 10 06/30 Flush 07/30 Flush 08/30 Write block 452 len 16384 data: 8 8 12 10 09/30 Flush 10/30 Write block 27 len 40960 data: 9 10 11 12 12 11 9 10 9 12 11/30 Flush 12/30 Write block 210 len 4096 data: 17 13/30 Read block 141 len 20480 14/30 Write block 486 len 36864 data: 12 13 12 10 11 10 10 8 11 15/30 Write block 559 len 8192 data: 11 12 16/30 Read block 488 len 16384 17/30 Write block 523 len 36864 data: 11 9 9 8 9 11 13 12 13 18/30 Flush 19/30 Write block 538 len 28672 data: 16 16 14 14 16 13 11 20/30 Flush 21/30 Write block 414 len 4096 data: 10 22/30 Read block 14 len 28672 23/30 Read block 427 len 16384 24/30 Read block 274 len 36864 25/30 Write block 153 len 36864 data: 12 13 11 11 13 14 15 16 18 26/30 Read block 39 len 12288 27/30 Write block 564 len 40960 data: 16 13 13 15 13 12 15 13 13 11 28/30 Write block 362 len 40960 data: 18 15 15 15 12 13 13 11 12 12 29/30 Read block 116 len 16384 30/30 Read block 429 len 4096 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Sent false 1001 Acked Read 9 Done Done Sent false 1002 Acked Write 10 Done Done Sent false 1003 Acked Read 9 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 4 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 7 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Write 10 Sent Done Sent false 1028 Acked Read 4 Sent Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 0 30 33 Done 27 30 0 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:27.461887657Z","hostname":"bmat-EVT22200007-0000ead9","pid":1859} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 164 294 294 291 F F F <--- 1 020-039 166 166 164 297 297 291 F F F <--- 2 040-059 160 160 160 282 282 282 F F F 3 060-079 165 165 165 293 293 293 F F F 4 080-099 163 163 163 288 288 288 F F F 5 100-119 165 165 165 293 293 293 F F F 6 120-139 165 165 165 293 293 293 F F F 7 140-159 164 164 164 289 289 289 T T F <--- 8 160-179 162 162 162 287 287 287 T T F <--- 9 180-199 158 158 158 279 279 279 F F F 10 200-219 166 166 162 298 298 286 F F F <--- 11 220-239 165 165 165 293 293 293 F F F 12 240-259 165 165 165 293 293 293 F F F 13 260-279 161 161 161 284 284 284 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 165 165 165 293 293 293 F F F 16 320-339 162 162 162 287 287 287 F F F 17 340-359 165 165 165 293 293 293 F F F 18 360-379 166 166 162 294 294 286 T T F <--- 19 380-399 153 153 153 273 273 273 F F F 20 400-419 165 165 165 293 293 293 T T F <--- 21 420-439 162 162 162 287 287 287 F F F 22 440-459 166 166 162 296 296 285 F F F <--- 23 460-479 153 153 153 273 273 273 F F F 24 480-499 166 166 164 298 298 291 F F F <--- 25 500-519 161 161 161 284 284 284 F F F 26 520-539 166 166 164 299 299 292 F F F <--- 27 540-559 166 166 158 299 299 279 F F F <--- 28 560-579 166 166 163 298 298 288 T T F <--- 29 580-599 165 165 165 293 293 293 F F F Max gen: 166, Max flush: 299 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 294 294 294 F F F 1 020-039 166 166 166 297 297 297 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 165 165 165 293 293 293 F F F 4 080-099 163 163 163 288 288 288 F F F 5 100-119 165 165 165 293 293 293 F F F 6 120-139 165 165 165 293 293 293 F F F 7 140-159 167 167 167 300 300 300 F F F 8 160-179 167 167 167 300 300 300 F F F 9 180-199 158 158 158 279 279 279 F F F 10 200-219 166 166 166 298 298 298 F F F 11 220-239 165 165 165 293 293 293 F F F 12 240-259 165 165 165 293 293 293 F F F 13 260-279 161 161 161 284 284 284 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 165 165 165 293 293 293 F F F 16 320-339 162 162 162 287 287 287 F F F 17 340-359 165 165 165 293 293 293 F F F 18 360-379 167 167 167 300 300 300 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 F F F 21 420-439 162 162 162 287 287 287 F F F 22 440-459 166 166 166 296 296 296 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 166 166 166 298 298 298 F F F 25 500-519 161 161 161 284 284 284 F F F 26 520-539 166 166 166 299 299 299 F F F 27 540-559 166 166 166 299 299 299 F F F 28 560-579 167 167 167 300 300 300 F F F 29 580-599 165 165 165 293 293 293 F F F Max gen: 167, Max flush: 300 Begin loop 84 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.113904806Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.139497666Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.140291137Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.140354553Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.140377158Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.140409792Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867} {"msg":"Crucible 289fccce-671d-412c-8116-d7d1d98167ce has session id: e7a77f26-ba0c-4eda-b450-18f5306ede76","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.141560891Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"Upstairs opts: Upstairs UUID: 289fccce-671d-412c-8116-d7d1d98167ce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.141665919Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"Crucible stats registered with UUID: 289fccce-671d-412c-8116-d7d1d98167ce","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.141697191Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:32.141722972Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.142155759Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.142230226Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.142505562Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.142776478Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.142833681Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867} {"msg":"289fccce-671d-412c-8116-d7d1d98167ce active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.142888219Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.14293898Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"client":"0","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.142995502Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"client":"1","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.143050571Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"client":"2","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.14310069Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.14315132Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.143205728Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"ds_connection connected from Ok(127.0.0.1:46506)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.143253072Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"io task","client":"0","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"ds_connection connected from Ok(127.0.0.1:60240)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.143679967Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"io task","client":"2","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"ds_connection connected from Ok(127.0.0.1:47611)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.143765636Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"io task","client":"1","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.145705547Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"client":"0","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.145784664Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"client":"2","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.145818711Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"client":"1","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.146609727Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.146767189Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"[0]R flush_numbers[0..12]: [294, 297, 282, 293, 288, 293, 293, 300, 300, 279, 298, 293]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.146916775Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"downstairs","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"[0]R generation[0..12]: [166, 166, 160, 165, 163, 165, 165, 167, 167, 158, 166, 165]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.146977986Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"downstairs","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147024929Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"downstairs","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"[1]R flush_numbers[0..12]: [294, 297, 282, 293, 288, 293, 293, 300, 300, 279, 298, 293]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147065139Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"downstairs","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"[1]R generation[0..12]: [166, 166, 160, 165, 163, 165, 165, 167, 167, 158, 166, 165]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147094818Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"downstairs","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147124196Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"downstairs","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"[2]R flush_numbers[0..12]: [294, 297, 282, 293, 288, 293, 293, 300, 300, 279, 298, 293]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147152903Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"downstairs","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"[2]R generation[0..12]: [166, 166, 160, 165, 163, 165, 165, 167, 167, 158, 166, 165]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147180708Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"downstairs","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-04T01:16:32.147210186Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"downstairs","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147250366Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"downstairs","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147275515Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"downstairs","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147299172Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"downstairs","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147322959Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"":"downstairs","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147347889Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147370584Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"client":"0","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147400202Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"client":"1","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147425613Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"client":"2","":"downstairs","session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147451394Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.14747494Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"289fccce-671d-412c-8116-d7d1d98167ce is now active with session: e7a77f26-ba0c-4eda-b450-18f5306ede76","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147501543Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147523797Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867,"session_id":"e7a77f26-ba0c-4eda-b450-18f5306ede76"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:32.147547344Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867} {"msg":"Activated sub_volume 289fccce-671d-412c-8116-d7d1d98167ce","v":0,"name":"crucible","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 level":30,"time":"2026-03-04T01:16:32.147589748Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 516 len 40960 data: 10 9 9 10 8 9 11 12 10 10 02/30 Write block 522 len 36864 data: 12 13 11 11 9 10 12 14 13 03/30 Write block 82 len 16384 data: 12 11 14 14 04/30 Write block 129 len 12288 data: 16 15 12 05/30 Flush 06/30 Write block 523 len 36864 data: 14 12 12 10 11 13 15 14 14 07/30 Write block 273 len 32768 data: 14 16 15 15 16 16 15 15 08/30 Read block 512 len 28672 09/30 Read block 169 len 36864 10/30 Read block 341 len 8192 11/30 Write block 97 len 24576 data: 19 16 15 15 7 8 12/30 Write block 256 len 24576 data: 9 13 13 14 17 18 13/30 Write block 121 len 8192 data: 12 10 14/30 Flush 15/30 Write block 197 len 40960 data: 14 15 15 18 17 17 16 14 18 19 16/30 Write block 588 len 36864 data: 8 11 12 9 10 9 10 11 12 17/30 Read block 574 len 36864 18/30 Write block 85 len 32768 data: 15 15 15 13 13 10 10 12 19/30 Read block 323 len 20480 20/30 Read block 585 len 8192 21/30 Read block 295 len 8192 22/30 Read block 65 len 24576 23/30 Read block 479 len 24576 24/30 Read block 574 len 36864 25/30 Read block 191 len 40960 26/30 Write block 108 len 16384 data: 11 10 13 14 27/30 Write block 315 len 36864 data: 11 13 10 10 11 16 14 16 16 28/30 Read block 526 len 20480 29/30 Read block 519 len 8192 30/30 Read block 510 len 36864 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 10 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 16 16 Done 16 16 0 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:32.181641886Z","hostname":"bmat-EVT22200007-0000ead9","pid":1867} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 294 294 294 F F F 1 020-039 166 166 166 297 297 297 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 165 165 165 293 293 293 F F F 4 080-099 168 168 168 302 302 302 T T F <--- 5 100-119 168 168 168 302 302 302 T T F <--- 6 120-139 168 168 168 302 302 302 F F F 7 140-159 167 167 167 300 300 300 F F F 8 160-179 167 167 167 300 300 300 F F F 9 180-199 158 158 158 279 279 279 T T F <--- 10 200-219 166 166 166 298 298 298 T T F <--- 11 220-239 165 165 165 293 293 293 F F F 12 240-259 168 168 168 302 302 302 F F F 13 260-279 168 168 168 302 302 302 F F F 14 280-299 168 168 168 302 302 302 F F F 15 300-319 165 165 165 293 293 293 T T F <--- 16 320-339 162 162 162 287 287 287 T T F <--- 17 340-359 165 165 165 293 293 293 F F F 18 360-379 167 167 167 300 300 300 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 F F F 21 420-439 162 162 162 287 287 287 F F F 22 440-459 166 166 166 296 296 296 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 166 166 166 298 298 298 F F F 25 500-519 168 168 168 301 301 301 F F F 26 520-539 168 168 168 302 302 302 F F F 27 540-559 166 166 166 299 299 299 F F F 28 560-579 167 167 167 300 300 300 F F F 29 580-599 165 165 165 293 293 293 T T F <--- Max gen: 168, Max flush: 302 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 294 294 294 F F F 1 020-039 166 166 166 297 297 297 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 165 165 165 293 293 293 F F F 4 080-099 169 169 169 303 303 303 F F F 5 100-119 169 169 169 303 303 303 F F F 6 120-139 168 168 168 302 302 302 F F F 7 140-159 167 167 167 300 300 300 F F F 8 160-179 167 167 167 300 300 300 F F F 9 180-199 169 169 169 303 303 303 F F F 10 200-219 169 169 169 303 303 303 F F F 11 220-239 165 165 165 293 293 293 F F F 12 240-259 168 168 168 302 302 302 F F F 13 260-279 168 168 168 302 302 302 F F F 14 280-299 168 168 168 302 302 302 F F F 15 300-319 169 169 169 303 303 303 F F F 16 320-339 169 169 169 303 303 303 F F F 17 340-359 165 165 165 293 293 293 F F F 18 360-379 167 167 167 300 300 300 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 F F F 21 420-439 162 162 162 287 287 287 F F F 22 440-459 166 166 166 296 296 296 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 166 166 166 298 298 298 F F F 25 500-519 168 168 168 301 301 301 F F F 26 520-539 168 168 168 302 302 302 F F F 27 540-559 166 166 166 299 299 299 F F F 28 560-579 167 167 167 300 300 300 F F F 29 580-599 169 169 169 303 303 303 F F F Max gen: 169, Max flush: 303 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.804821963Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:36.834279966Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.834370595Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.866423405Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.866982051Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.867064194Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.867098131Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.867145284Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875} {"msg":"Crucible a724f1a4-5d62-4372-ac5d-67da893d9d0c has session id: 2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.868292636Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"Upstairs opts: Upstairs UUID: a724f1a4-5d62-4372-ac5d-67da893d9d0c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.868351963Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"Crucible stats registered with UUID: a724f1a4-5d62-4372-ac5d-67da893d9d0c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.868389327Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:36.868414708Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.868865089Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.868919818Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.868948545Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.869344228Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.869389427Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875} {"msg":"a724f1a4-5d62-4372-ac5d-67da893d9d0c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.86949174Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.86952203Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"client":"0","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.869549124Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"client":"1","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.869575877Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"client":"2","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.869602469Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.869712197Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"ds_connection connected from Ok(127.0.0.1:49719)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.869749761Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"io task","client":"0","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.869777476Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59724)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.869810852Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"io task","client":"2","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"ds_connection connected from Ok(127.0.0.1:50123)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.870124793Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"io task","client":"1","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.871934767Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"client":"2","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.87202219Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"client":"1","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.872056107Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"client":"0","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.872828897Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.872864999Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"[0]R flush_numbers[0..12]: [294, 297, 282, 293, 303, 303, 302, 300, 300, 303, 303, 293]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.872955818Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"downstairs","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"[0]R generation[0..12]: [166, 166, 160, 165, 169, 169, 168, 167, 167, 169, 169, 165]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.87299207Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"downstairs","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873024655Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"downstairs","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"[1]R flush_numbers[0..12]: [294, 297, 282, 293, 303, 303, 302, 300, 300, 303, 303, 293]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873055035Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"downstairs","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"[1]R generation[0..12]: [166, 166, 160, 165, 169, 169, 168, 167, 167, 169, 169, 165]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873085936Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"downstairs","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873117428Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"downstairs","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"[2]R flush_numbers[0..12]: [294, 297, 282, 293, 303, 303, 302, 300, 300, 303, 303, 293]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873148169Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"downstairs","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"[2]R generation[0..12]: [166, 166, 160, 165, 169, 169, 168, 167, 167, 169, 169, 165]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.87317876Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"downstairs","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873218028Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"downstairs","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.87324431Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"downstairs","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873271303Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"downstairs","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873297214Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"downstairs","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873322885Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"":"downstairs","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873349708Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873374137Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"client":"0","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.87340096Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"client":"1","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873427472Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"client":"2","":"downstairs","session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"All required reconciliation work is completed","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2026-03-04T01:16:36.873465758Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873492391Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"a724f1a4-5d62-4372-ac5d-67da893d9d0c is now active with session: 2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873520847Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873544754Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875,"session_id":"2ef912a0-0ff1-4c0e-99ee-d7c8064f5b5d"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873570035Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875} {"msg":"Activated sub_volume a724f1a4-5d62-4372-ac5d-67da893d9d0c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:36.873592078Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 155 len 16384 data: 12 12 14 15 02/30 Write block 552 len 20480 data: 13 12 14 15 14 03/30 Read block 179 len 40960 04/30 Read block 295 len 20480 05/30 Flush 06/30 Write block 326 len 20480 data: 11 9 9 9 10 07/30 Read block 138 len 8192 08/30 Read block 92 len 40960 09/30 Read block 228 len 40960 10/30 Read block 130 len 28672 11/30 Flush 12/30 Write block 361 len 24576 data: 16 19 16 16 16 13 13/30 Read block 522 len 8192 14/30 Read block 280 len 8192 15/30 Write block 509 len 24576 data: 11 13 11 12 11 13 16/30 Read block 68 len 8192 17/30 Read block 314 len 20480 18/30 Read block 338 len 24576 19/30 Flush 20/30 Write block 237 len 32768 data: 16 17 19 18 15 12 11 9 21/30 Read block 125 len 36864 22/30 Write block 505 len 12288 data: 10 8 9 23/30 Write block 62 len 4096 data: 8 24/30 Read block 422 len 40960 25/30 Read block 93 len 8192 26/30 Read block 21 len 36864 27/30 Write block 4 len 8192 data: 14 16 28/30 Write block 353 len 20480 data: 7 4 6 9 14 29/30 Read block 14 len 24576 30/30 Read block 548 len 36864 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Sent Done Done false 1001 Acked Write 5 Sent Done Done false 1002 Acked Read 10 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Write 6 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:36.907642233Z","hostname":"bmat-EVT22200007-0000ead9","pid":1875} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 294 294 294 F T T <--- 1 020-039 166 166 166 297 297 297 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 165 165 165 293 293 293 F T T <--- 4 080-099 169 169 169 303 303 303 F F F 5 100-119 169 169 169 303 303 303 F F F 6 120-139 168 168 168 302 302 302 F F F 7 140-159 167 170 170 300 304 304 F F F <--- 8 160-179 167 167 167 300 300 300 F F F 9 180-199 169 169 169 303 303 303 F F F 10 200-219 169 169 169 303 303 303 F F F 11 220-239 165 165 165 293 293 293 F T T <--- 12 240-259 168 168 168 302 302 302 F T T <--- 13 260-279 168 168 168 302 302 302 F F F 14 280-299 168 168 168 302 302 302 F F F 15 300-319 169 169 169 303 303 303 F F F 16 320-339 169 170 170 303 305 305 F F F <--- 17 340-359 165 165 165 293 293 293 F T T <--- 18 360-379 167 170 170 300 306 306 F F F <--- 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 F F F 21 420-439 162 162 162 287 287 287 F F F 22 440-459 166 166 166 296 296 296 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 166 166 166 298 298 298 F F F 25 500-519 168 170 170 301 306 306 F T T <--- 26 520-539 168 168 168 302 302 302 F F F 27 540-559 166 170 170 299 304 304 F F F <--- 28 560-579 167 167 167 300 300 300 F F F 29 580-599 169 169 169 303 303 303 F F F Max gen: 170, Max flush: 306 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 307 307 307 F F F 1 020-039 166 166 166 297 297 297 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 171 171 171 307 307 307 F F F 4 080-099 169 169 169 303 303 303 F F F 5 100-119 169 169 169 303 303 303 F F F 6 120-139 168 168 168 302 302 302 F F F 7 140-159 170 170 170 304 304 304 F F F 8 160-179 167 167 167 300 300 300 F F F 9 180-199 169 169 169 303 303 303 F F F 10 200-219 169 169 169 303 303 303 F F F 11 220-239 171 171 171 307 307 307 F F F 12 240-259 171 171 171 307 307 307 F F F 13 260-279 168 168 168 302 302 302 F F F 14 280-299 168 168 168 302 302 302 F F F 15 300-319 169 169 169 303 303 303 F F F 16 320-339 170 170 170 305 305 305 F F F 17 340-359 171 171 171 307 307 307 F F F 18 360-379 170 170 170 306 306 306 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 F F F 21 420-439 162 162 162 287 287 287 F F F 22 440-459 166 166 166 296 296 296 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 166 166 166 298 298 298 F F F 25 500-519 171 171 171 307 307 307 F F F 26 520-539 168 168 168 302 302 302 F F F 27 540-559 170 170 170 304 304 304 F F F 28 560-579 167 167 167 300 300 300 F F F 29 580-599 169 169 169 303 303 303 F F F Max gen: 171, Max flush: 307 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.563142227Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.593956265Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.594634318Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.594766419Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.594803553Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.594859844Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883} {"msg":"Crucible 759041b4-eb18-4f77-8d81-47317e4f8a61 has session id: 007ef5df-ee31-48cf-bfdc-ce8f8d595ed7","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.596055561Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"Upstairs opts: Upstairs UUID: 759041b4-eb18-4f77-8d81-47317e4f8a61, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.596132554Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"Crucible stats registered with UUID: 759041b4-eb18-4f77-8d81-47317e4f8a61","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.596168555Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:41.596200959Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.596767842Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.59683244Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.596868701Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.59717613Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.597224065Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883} {"msg":"759041b4-eb18-4f77-8d81-47317e4f8a61 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.597361747Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.597406897Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"client":"0","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.597440564Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"client":"1","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.597473779Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"client":"2","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.597506304Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.597540321Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.597577024Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"ds_connection connected from Ok(127.0.0.1:38983)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.597965673Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"io task","client":"1","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"ds_connection connected from Ok(127.0.0.1:34755)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.598203384Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"io task","client":"2","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"ds_connection connected from Ok(127.0.0.1:59369)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.5982669Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"io task","client":"0","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.599979921Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"client":"2","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.600281218Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"client":"1","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.60062575Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"client":"0","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.600735557Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601113976Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"[0]R flush_numbers[0..12]: [307, 297, 282, 307, 303, 303, 302, 304, 300, 303, 303, 307]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601294474Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"downstairs","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"[0]R generation[0..12]: [171, 166, 160, 171, 169, 169, 168, 170, 167, 169, 169, 171]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601342379Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"downstairs","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601397868Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"downstairs","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"[1]R flush_numbers[0..12]: [307, 297, 282, 307, 303, 303, 302, 304, 300, 303, 303, 307]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601453038Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"downstairs","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"[1]R generation[0..12]: [171, 166, 160, 171, 169, 169, 168, 170, 167, 169, 169, 171]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601504229Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"downstairs","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601547565Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"downstairs","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"[2]R flush_numbers[0..12]: [307, 297, 282, 307, 303, 303, 302, 304, 300, 303, 303, 307]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601597484Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"downstairs","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"[2]R generation[0..12]: [171, 166, 160, 171, 169, 169, 168, 170, 167, 169, 169, 171]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601647623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"downstairs","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601719736Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"downstairs","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601773352Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"downstairs","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601828902Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"downstairs","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601870093Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"downstairs","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601904932Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.601957927Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.602003067Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"client":"0","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.602048106Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"client":"1","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.6021108Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"client":"2","":"downstairs","session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.602172652Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-04T01:16:41.602229365Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"759041b4-eb18-4f77-8d81-47317e4f8a61 is now active with session: 007ef5df-ee31-48cf-bfdc-ce8f8d595ed7","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.602293391Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.602346757Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883,"session_id":"007ef5df-ee31-48cf-bfdc-ce8f8d595ed7"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.602402497Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883} {"msg":"Activated sub_volume 759041b4-eb18-4f77-8d81-47317e4f8a61","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:41.602450222Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 131 len 4096 data: 13 02/30 Write block 561 len 40960 data: 11 14 16 17 14 14 16 14 13 16 03/30 Flush 04/30 Write block 183 len 40960 data: 12 13 13 14 14 15 12 13 16 15 05/30 Read block 199 len 36864 06/30 Read block 305 len 32768 07/30 Write block 521 len 8192 data: 10 13 08/30 Read block 504 len 8192 09/30 Write block 511 len 4096 data: 12 10/30 Read block 85 len 8192 11/30 Read block 485 len 8192 12/30 Read block 319 len 40960 13/30 Write block 552 len 20480 data: 14 13 15 16 15 14/30 Read block 393 len 16384 15/30 Write block 524 len 40960 data: 13 13 11 12 14 16 15 15 14 17 16/30 Write block 148 len 4096 data: 14 17/30 Write block 81 len 16384 data: 10 13 12 15 18/30 Read block 375 len 24576 19/30 Write block 337 len 24576 data: 14 12 12 13 14 16 20/30 Read block 35 len 8192 21/30 Read block 76 len 36864 22/30 Write block 117 len 16384 data: 21 19 16 16 23/30 Write block 501 len 4096 data: 9 24/30 Write block 89 len 8192 data: 14 11 25/30 Write block 445 len 32768 data: 15 17 17 17 16 12 10 9 26/30 Flush 27/30 Write block 299 len 32768 data: 9 10 9 12 14 15 15 15 28/30 Write block 277 len 12288 data: 17 17 16 29/30 Write block 581 len 16384 data: 17 16 16 18 30/30 Write block 291 len 16384 data: 17 16 11 13 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 10 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Read 8 Done Sent Done false 1006 Acked Write 2 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Write 5 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:41.637798166Z","hostname":"bmat-EVT22200007-0000ead9","pid":1883} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 307 307 307 F F F 1 020-039 166 166 166 297 297 297 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 171 171 171 307 307 307 F F F 4 080-099 172 169 172 309 303 309 F F F <--- 5 100-119 172 169 172 309 303 309 F F F <--- 6 120-139 172 168 172 309 302 309 F T F <--- 7 140-159 172 170 172 309 304 309 F F F <--- 8 160-179 167 167 167 300 300 300 F F F 9 180-199 172 169 172 309 303 309 F F F <--- 10 200-219 169 169 169 303 303 303 F F F 11 220-239 171 171 171 307 307 307 F F F 12 240-259 171 171 171 307 307 307 F F F 13 260-279 168 168 168 302 302 302 T F T <--- 14 280-299 168 168 168 302 302 302 T F T <--- 15 300-319 169 169 169 303 303 303 T F T <--- 16 320-339 172 170 172 309 305 309 F F F <--- 17 340-359 172 171 172 309 307 309 F F F <--- 18 360-379 170 170 170 306 306 306 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 F F F 21 420-439 162 162 162 287 287 287 F F F 22 440-459 172 166 172 309 296 309 F F F <--- 23 460-479 153 153 153 273 273 273 F F F 24 480-499 166 166 166 298 298 298 F F F 25 500-519 172 171 172 309 307 309 F F F <--- 26 520-539 172 168 172 309 302 309 F F F <--- 27 540-559 172 170 172 309 304 309 F F F <--- 28 560-579 172 167 172 308 300 308 F F F <--- 29 580-599 169 169 169 303 303 303 T F T <--- Max gen: 172, Max flush: 309 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 307 307 307 F F F 1 020-039 166 166 166 297 297 297 F F F 2 040-059 160 160 160 282 282 282 F F F 3 060-079 171 171 171 307 307 307 F F F 4 080-099 172 172 172 309 309 309 F F F 5 100-119 172 172 172 309 309 309 F F F 6 120-139 172 172 172 309 309 309 F F F 7 140-159 172 172 172 309 309 309 F F F 8 160-179 167 167 167 300 300 300 F F F 9 180-199 172 172 172 309 309 309 F F F 10 200-219 169 169 169 303 303 303 F F F 11 220-239 171 171 171 307 307 307 F F F 12 240-259 171 171 171 307 307 307 F F F 13 260-279 173 173 173 310 310 310 F F F 14 280-299 173 173 173 310 310 310 F F F 15 300-319 173 173 173 310 310 310 F F F 16 320-339 172 172 172 309 309 309 F F F 17 340-359 172 172 172 309 309 309 F F F 18 360-379 170 170 170 306 306 306 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 F F F 21 420-439 162 162 162 287 287 287 F F F 22 440-459 172 172 172 309 309 309 F F F 23 460-479 153 153 153 273 273 273 F F F 24 480-499 166 166 166 298 298 298 F F F 25 500-519 172 172 172 309 309 309 F F F 26 520-539 172 172 172 309 309 309 F F F 27 540-559 172 172 172 309 309 309 F F F 28 560-579 172 172 172 308 308 308 F F F 29 580-599 173 173 173 310 310 310 F F F Max gen: 173, Max flush: 310 Begin loop 87 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.34451367Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.374744302Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.375436954Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.3755512Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.375588444Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.375649264Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891} {"msg":"Crucible f06b1755-816d-4e97-b6a6-a376fa2d0ec6 has session id: 50271583-1c3e-4719-b314-dbd475253f92","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.376865281Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"Upstairs opts: Upstairs UUID: f06b1755-816d-4e97-b6a6-a376fa2d0ec6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.376983305Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"Crucible stats registered with UUID: f06b1755-816d-4e97-b6a6-a376fa2d0ec6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.377037412Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:46.377078694Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.377701176Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.377815122Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.377852005Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.377897104Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.378025619Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891} {"msg":"f06b1755-816d-4e97-b6a6-a376fa2d0ec6 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.378063263Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.378095898Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"client":"0","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.378129875Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"client":"1","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.378162991Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"client":"2","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.378197028Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.378317116Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.378359079Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"ds_connection connected from Ok(127.0.0.1:61516)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.378850642Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"io task","client":"2","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"ds_connection connected from Ok(127.0.0.1:47856)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.379227799Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"io task","client":"0","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"ds_connection connected from Ok(127.0.0.1:52530)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.379308669Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"io task","client":"1","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.380871864Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"client":"1","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.381152059Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"client":"0","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.381468145Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"client":"2","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.381685174Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.381792196Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"[0]R flush_numbers[0..12]: [307, 297, 282, 307, 309, 309, 309, 309, 300, 309, 303, 307]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.382659002Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"downstairs","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"[0]R generation[0..12]: [171, 166, 160, 171, 172, 172, 172, 172, 167, 172, 169, 171]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.382737277Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"downstairs","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.382791404Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"downstairs","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"[1]R flush_numbers[0..12]: [307, 297, 282, 307, 309, 309, 309, 309, 300, 309, 303, 307]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.382847826Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"downstairs","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"[1]R generation[0..12]: [171, 166, 160, 171, 172, 172, 172, 172, 167, 172, 169, 171]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.382902875Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"downstairs","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.382967122Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"downstairs","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"[2]R flush_numbers[0..12]: [307, 297, 282, 307, 309, 309, 309, 309, 300, 309, 303, 307]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383060207Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"downstairs","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"[2]R generation[0..12]: [171, 166, 160, 171, 172, 172, 172, 172, 167, 172, 169, 171]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383115696Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"downstairs","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383181587Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383240253Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"downstairs","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383290032Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"downstairs","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383335221Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"downstairs","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383380501Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"":"downstairs","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383429758Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.38347118Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"client":"0","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383514977Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"client":"1","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383565036Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"client":"2","":"downstairs","session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383616748Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383661337Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"f06b1755-816d-4e97-b6a6-a376fa2d0ec6 is now active with session: 50271583-1c3e-4719-b314-dbd475253f92","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383710975Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383756435Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891,"session_id":"50271583-1c3e-4719-b314-dbd475253f92"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-04T01:16:46.383801825Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891} {"msg":"Activated sub_volume f06b1755-816d-4e97-b6a6-a376fa2d0ec6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:46.383842505Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 15 len 40960 data: 11 8 11 12 14 16 15 16 12 11 02/30 Read block 560 len 40960 03/30 Read block 259 len 36864 04/30 Write block 315 len 32768 data: 12 14 11 11 12 17 15 17 05/30 Read block 556 len 28672 06/30 Write block 486 len 36864 data: 13 14 13 11 12 11 11 9 12 07/30 Read block 298 len 40960 08/30 Read block 158 len 16384 09/30 Read block 78 len 8192 10/30 Read block 529 len 8192 11/30 Write block 522 len 20480 data: 14 15 14 14 12 12/30 Write block 427 len 4096 data: 11 13/30 Read block 386 len 4096 14/30 Read block 435 len 20480 15/30 Write block 212 len 20480 data: 13 10 9 8 11 16/30 Read block 88 len 20480 17/30 Write block 139 len 4096 data: 19 18/30 Write block 254 len 20480 data: 10 11 10 14 14 19/30 Read block 223 len 20480 20/30 Read block 402 len 28672 21/30 Write block 227 len 24576 data: 10 12 13 13 17 17 22/30 Read block 353 len 40960 23/30 Flush 24/30 Write block 358 len 4096 data: 19 25/30 Flush 26/30 Write block 476 len 28672 data: 10 9 8 10 12 11 13 27/30 Read block 308 len 36864 28/30 Read block 274 len 28672 29/30 Read block 279 len 20480 30/30 Write block 34 len 36864 data: 11 10 13 10 9 9 7 8 8 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Sent false 1001 Acked Read 10 Done Done Sent false 1002 Acked Read 9 Done Done Sent false 1003 Acked Write 8 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 7 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:46.423190229Z","hostname":"bmat-EVT22200007-0000ead9","pid":1891} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 171 311 311 307 F F F <--- 1 020-039 174 174 166 311 311 297 T T F <--- 2 040-059 160 160 160 282 282 282 T T F <--- 3 060-079 171 171 171 307 307 307 F F F 4 080-099 172 172 172 309 309 309 F F F 5 100-119 172 172 172 309 309 309 F F F 6 120-139 174 174 172 311 311 309 F F F <--- 7 140-159 172 172 172 309 309 309 F F F 8 160-179 167 167 167 300 300 300 F F F 9 180-199 172 172 172 309 309 309 F F F 10 200-219 174 174 169 311 311 303 F F F <--- 11 220-239 174 174 171 311 311 307 F F F <--- 12 240-259 174 174 171 311 311 307 F F F <--- 13 260-279 173 173 173 310 310 310 F F F 14 280-299 173 173 173 310 310 310 F F F 15 300-319 174 174 173 311 311 310 F F F <--- 16 320-339 174 174 172 311 311 309 F F F <--- 17 340-359 174 174 172 312 312 309 F F F <--- 18 360-379 170 170 170 306 306 306 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 F F F 21 420-439 174 174 162 311 311 287 F F F <--- 22 440-459 172 172 172 309 309 309 F F F 23 460-479 153 153 153 273 273 273 T T F <--- 24 480-499 174 174 166 311 311 298 T T F <--- 25 500-519 172 172 172 309 309 309 F F F 26 520-539 174 174 172 311 311 309 F F F <--- 27 540-559 172 172 172 309 309 309 F F F 28 560-579 172 172 172 308 308 308 F F F 29 580-599 173 173 173 310 310 310 F F F Max gen: 174, Max flush: 312 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 311 311 311 F F F 1 020-039 175 175 175 313 313 313 F F F 2 040-059 175 175 175 313 313 313 F F F 3 060-079 171 171 171 307 307 307 F F F 4 080-099 172 172 172 309 309 309 F F F 5 100-119 172 172 172 309 309 309 F F F 6 120-139 174 174 174 311 311 311 F F F 7 140-159 172 172 172 309 309 309 F F F 8 160-179 167 167 167 300 300 300 F F F 9 180-199 172 172 172 309 309 309 F F F 10 200-219 174 174 174 311 311 311 F F F 11 220-239 174 174 174 311 311 311 F F F 12 240-259 174 174 174 311 311 311 F F F 13 260-279 173 173 173 310 310 310 F F F 14 280-299 173 173 173 310 310 310 F F F 15 300-319 174 174 174 311 311 311 F F F 16 320-339 174 174 174 311 311 311 F F F 17 340-359 174 174 174 312 312 312 F F F 18 360-379 170 170 170 306 306 306 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 F F F 21 420-439 174 174 174 311 311 311 F F F 22 440-459 172 172 172 309 309 309 F F F 23 460-479 175 175 175 313 313 313 F F F 24 480-499 175 175 175 313 313 313 F F F 25 500-519 172 172 172 309 309 309 F F F 26 520-539 174 174 174 311 311 311 F F F 27 540-559 172 172 172 309 309 309 F F F 28 560-579 172 172 172 308 308 308 F F F 29 580-599 173 173 173 310 310 310 F F F Max gen: 175, Max flush: 313 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.093415627Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.120623235Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.121321637Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.121454741Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.121495983Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.121546102Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899} {"msg":"Crucible e5c28f6b-244d-4386-b910-cd0f5836d587 has session id: 8125bd12-d3fd-40f4-8d11-e732bde57048","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.12255537Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"Upstairs opts: Upstairs UUID: e5c28f6b-244d-4386-b910-cd0f5836d587, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.12263133Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"Crucible stats registered with UUID: e5c28f6b-244d-4386-b910-cd0f5836d587","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.122662111Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:51.122687552Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.123112733Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.123172061Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.123201429Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.12327259Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.123300034Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899} {"msg":"e5c28f6b-244d-4386-b910-cd0f5836d587 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.123484229Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.123521433Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"client":"0","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.123549188Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"client":"1","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.123576462Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"client":"2","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.123603465Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.12398454Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.124022364Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"ds_connection connected from Ok(127.0.0.1:46881)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.124066241Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"io task","client":"0","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"ds_connection connected from Ok(127.0.0.1:42666)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.124158764Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"io task","client":"2","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"ds_connection connected from Ok(127.0.0.1:48329)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.124191289Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"io task","client":"1","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.125561442Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"client":"1","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.126041763Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"client":"0","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.126244073Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.126407376Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"client":"2","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.126701328Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"[0]R flush_numbers[0..12]: [311, 313, 313, 307, 309, 309, 311, 309, 300, 309, 311, 311]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.12756055Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"downstairs","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"[0]R generation[0..12]: [174, 175, 175, 171, 172, 172, 174, 172, 167, 172, 174, 174]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127607102Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"downstairs","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127640197Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"downstairs","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"[1]R flush_numbers[0..12]: [311, 313, 313, 307, 309, 309, 311, 309, 300, 309, 311, 311]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127670898Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"downstairs","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"[1]R generation[0..12]: [174, 175, 175, 171, 172, 172, 174, 172, 167, 172, 174, 174]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127701829Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"downstairs","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127733883Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"downstairs","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"[2]R flush_numbers[0..12]: [311, 313, 313, 307, 309, 309, 311, 309, 300, 309, 311, 311]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127764453Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"downstairs","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"[2]R generation[0..12]: [174, 175, 175, 171, 172, 172, 174, 172, 167, 172, 174, 174]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127794503Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"downstairs","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127826376Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"downstairs","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127852668Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"downstairs","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127879641Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"downstairs","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127905062Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"downstairs","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127930652Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"":"downstairs","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127957385Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.127982184Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"client":"0","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.128009358Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"client":"1","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.128042083Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"client":"2","":"downstairs","session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.128069708Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.128095288Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"e5c28f6b-244d-4386-b910-cd0f5836d587 is now active with session: 8125bd12-d3fd-40f4-8d11-e732bde57048","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.128124135Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.128148283Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899,"session_id":"8125bd12-d3fd-40f4-8d11-e732bde57048"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.128174064Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899} {"msg":"Activated sub_volume e5c28f6b-244d-4386-b910-cd0f5836d587","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:51.128196539Z","hostname":"Wait for a query_work_queue command to finish before sending IO bmat-EVT22200007-0000ead9","pid":1899} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 535 len 4096 data: 14 02/30 Write block 25 len 36864 data: 8 6 10 11 12 13 13 12 10 03/30 Write block 463 len 28672 data: 17 15 14 15 15 12 13 04/30 Read block 318 len 40960 05/30 Read block 393 len 32768 06/30 Read block 396 len 40960 07/30 Write block 592 len 20480 data: 11 10 11 12 13 08/30 Read block 263 len 40960 09/30 Write block 288 len 28672 data: 14 13 13 18 17 12 14 10/30 Write block 550 len 28672 data: 10 12 15 14 16 17 16 11/30 Write block 574 len 28672 data: 12 15 13 15 18 17 19 12/30 Read block 483 len 12288 13/30 Write block 192 len 4096 data: 16 14/30 Flush 15/30 Flush 16/30 Write block 104 len 4096 data: 7 17/30 Write block 553 len 8192 data: 15 17 18/30 Write block 308 len 28672 data: 17 18 16 15 12 11 9 19/30 Write block 7 len 24576 data: 18 13 13 12 14 13 20/30 Read block 73 len 20480 21/30 Write block 40 len 28672 data: 8 9 9 13 13 16 12 22/30 Flush 23/30 Write block 199 len 16384 data: 16 19 18 18 24/30 Read block 591 len 32768 25/30 Flush 26/30 Write block 37 len 36864 data: 11 10 10 9 10 10 14 14 17 27/30 Read block 523 len 32768 28/30 Read block 565 len 36864 29/30 Read block 487 len 36864 30/30 Read block 581 len 32768 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Read 8 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Write 2 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 8 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 26 26 Done 30 30 4 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:51.166519964Z","hostname":"bmat-EVT22200007-0000ead9","pid":1899} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 174 316 316 311 F F F <--- 1 020-039 176 176 175 314 314 313 T T T <--- 2 040-059 176 176 175 316 316 313 T T F <--- 3 060-079 171 171 171 307 307 307 F F F 4 080-099 172 172 172 309 309 309 F F F 5 100-119 176 176 172 316 316 309 F F F <--- 6 120-139 174 174 174 311 311 311 F F F 7 140-159 172 172 172 309 309 309 F F F 8 160-179 167 167 167 300 300 300 F F F 9 180-199 176 176 172 317 317 309 F F F <--- 10 200-219 176 176 174 317 317 311 F F F <--- 11 220-239 174 174 174 311 311 311 F F F 12 240-259 174 174 174 311 311 311 F F F 13 260-279 173 173 173 310 310 310 F F F 14 280-299 176 176 173 314 314 310 F F F <--- 15 300-319 176 176 174 316 316 311 F F F <--- 16 320-339 174 174 174 311 311 311 F F F 17 340-359 174 174 174 312 312 312 F F F 18 360-379 170 170 170 306 306 306 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 F F F 21 420-439 174 174 174 311 311 311 F F F 22 440-459 172 172 172 309 309 309 F F F 23 460-479 176 176 175 314 314 313 F F T <--- 24 480-499 175 175 175 313 313 313 F F F 25 500-519 172 172 172 309 309 309 F F F 26 520-539 176 176 174 314 314 311 F F T <--- 27 540-559 176 176 172 316 316 309 F F F <--- 28 560-579 176 176 172 314 314 308 F F F <--- 29 580-599 176 176 173 314 314 310 F F F <--- Max gen: 176, Max flush: 317 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 176 176 316 316 316 F F F 1 020-039 177 177 177 318 318 318 F F F 2 040-059 177 177 177 318 318 318 F F F 3 060-079 171 171 171 307 307 307 F F F 4 080-099 172 172 172 309 309 309 F F F 5 100-119 176 176 176 316 316 316 F F F 6 120-139 174 174 174 311 311 311 F F F 7 140-159 172 172 172 309 309 309 F F F 8 160-179 167 167 167 300 300 300 F F F 9 180-199 176 176 176 317 317 317 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 174 174 174 311 311 311 F F F 12 240-259 174 174 174 311 311 311 F F F 13 260-279 173 173 173 310 310 310 F F F 14 280-299 176 176 176 314 314 314 F F F 15 300-319 176 176 176 316 316 316 F F F 16 320-339 174 174 174 311 311 311 F F F 17 340-359 174 174 174 312 312 312 F F F 18 360-379 170 170 170 306 306 306 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 F F F 21 420-439 174 174 174 311 311 311 F F F 22 440-459 172 172 172 309 309 309 F F F 23 460-479 176 176 176 314 314 314 F F F 24 480-499 175 175 175 313 313 313 F F F 25 500-519 172 172 172 309 309 309 F F F 26 520-539 176 176 176 314 314 314 F F F 27 540-559 176 176 176 316 316 316 F F F 28 560-579 176 176 176 314 314 314 F F F 29 580-599 176 176 176 314 314 314 F F F Max gen: 177, Max flush: 318 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.848061576Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:55.873022886Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.873140189Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.901090386Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.901874459Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.901979416Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.902032722Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.902068994Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907} {"msg":"Crucible 218d8361-3c6f-4532-bb04-2830cac3fbf5 has session id: e91a678d-0ec2-418d-8daf-d644a366a2d5","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.903590025Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"Upstairs opts: Upstairs UUID: 218d8361-3c6f-4532-bb04-2830cac3fbf5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.903790873Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"Crucible stats registered with UUID: 218d8361-3c6f-4532-bb04-2830cac3fbf5","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.903837836Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:55.903871603Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.904299129Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.90437037Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.904408435Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.904589624Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.904632088Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907} {"msg":"218d8361-3c6f-4532-bb04-2830cac3fbf5 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.904747447Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.904786825Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"client":"0","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.904824138Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"client":"1","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.904861102Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"client":"2","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.904896822Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.904934517Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.905116177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:43431)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.905172548Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"io task","client":"2","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:51484)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.905214862Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"io task","client":"0","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:51499)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.905455919Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"io task","client":"1","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.907387393Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"client":"1","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.90778451Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"client":"0","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.908146737Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.908257175Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"client":"2","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.908687537Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"[0]R flush_numbers[0..12]: [316, 318, 318, 307, 309, 316, 311, 309, 300, 317, 317, 311]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909080285Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"downstairs","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"[0]R generation[0..12]: [176, 177, 177, 171, 172, 176, 174, 172, 167, 176, 176, 174]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909120835Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"downstairs","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909156386Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"downstairs","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"[1]R flush_numbers[0..12]: [316, 318, 318, 307, 309, 316, 311, 309, 300, 317, 317, 311]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909190052Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"downstairs","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"[1]R generation[0..12]: [176, 177, 177, 171, 172, 176, 174, 172, 167, 176, 176, 174]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909223689Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"downstairs","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909258528Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"downstairs","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"[2]R flush_numbers[0..12]: [316, 318, 318, 307, 309, 316, 311, 309, 300, 317, 317, 311]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.90929997Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"downstairs","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"[2]R generation[0..12]: [176, 177, 177, 171, 172, 176, 174, 172, 167, 176, 176, 174]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909333767Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"downstairs","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909368405Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"downstairs","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909397032Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"downstairs","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909426991Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"downstairs","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909454636Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"downstairs","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909482662Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"":"downstairs","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.90951211Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909539304Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"client":"0","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909569063Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"client":"1","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909610705Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"client":"2","":"downstairs","session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909641145Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909669311Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"218d8361-3c6f-4532-bb04-2830cac3fbf5 is now active with session: e91a678d-0ec2-418d-8daf-d644a366a2d5","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909701234Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909727957Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907,"session_id":"e91a678d-0ec2-418d-8daf-d644a366a2d5"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909759219Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907} {"msg":"Activated sub_volume 218d8361-3c6f-4532-bb04-2830cac3fbf5","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:16:55.909783177Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 195 len 16384 data: 13 15 15 16 02/30 Flush 03/30 Write block 475 len 40960 data: 13 11 10 9 11 13 12 14 12 11 04/30 Write block 269 len 28672 data: 13 15 15 15 15 17 16 05/30 Write block 436 len 24576 data: 19 22 21 17 16 12 06/30 Write block 120 len 36864 data: 17 13 11 9 10 10 14 15 14 07/30 Read block 112 len 4096 08/30 Read block 588 len 8192 09/30 Read block 173 len 20480 10/30 Read block 248 len 12288 11/30 Read block 486 len 4096 12/30 Write block 2 len 16384 data: 8 12 15 17 13/30 Read block 142 len 20480 14/30 Write block 482 len 24576 data: 15 13 12 11 14 15 15/30 Write block 236 len 12288 data: 21 17 18 16/30 Write block 518 len 28672 data: 10 11 9 11 15 16 15 17/30 Flush 18/30 Flush 19/30 Write block 268 len 28672 data: 13 14 16 16 16 16 18 20/30 Write block 232 len 40960 data: 18 18 20 19 22 18 19 20 19 16 21/30 Read block 413 len 24576 22/30 Write block 359 len 4096 data: 18 23/30 Read block 477 len 32768 24/30 Read block 473 len 16384 25/30 Flush 26/30 Flush 27/30 Write block 143 len 24576 data: 11 11 10 13 15 15 28/30 Write block 253 len 8192 data: 9 11 29/30 Write block 357 len 36864 data: 15 20 19 16 17 20 17 17 17 30/30 Read block 595 len 4096 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 10 Sent Done Done false 1003 Acked Write 7 Sent Done Done false 1004 Acked Write 6 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Write 9 Sent Sent Done false 1029 Acked Read 1 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 2 0 30 Done 0 26 28 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:16:55.947083977Z","hostname":"bmat-EVT22200007-0000ead9","pid":1907} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 176 178 178 316 320 320 F F F <--- 1 020-039 177 177 177 318 318 318 F F F 2 040-059 177 177 177 318 318 318 F F F 3 060-079 171 171 171 307 307 307 F F F 4 080-099 172 172 172 309 309 309 F F F 5 100-119 176 176 176 316 316 316 F F F 6 120-139 174 178 178 311 320 320 F F F <--- 7 140-159 172 172 172 309 309 309 F T T <--- 8 160-179 167 167 167 300 300 300 F F F 9 180-199 178 178 178 319 319 319 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 174 178 178 311 322 322 F F F <--- 12 240-259 174 178 178 311 322 322 F T T <--- 13 260-279 173 178 178 310 322 322 F F F <--- 14 280-299 176 176 176 314 314 314 F F F 15 300-319 176 176 176 316 316 316 F F F 16 320-339 174 174 174 311 311 311 F F F 17 340-359 174 178 178 312 322 322 F T T <--- 18 360-379 170 170 170 306 306 306 F T T <--- 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 F F F 21 420-439 174 178 178 311 320 320 F F F <--- 22 440-459 172 178 178 309 320 320 F F F <--- 23 460-479 176 178 178 314 320 320 F F F <--- 24 480-499 175 178 178 313 320 320 F F F <--- 25 500-519 172 178 178 309 320 320 F F F <--- 26 520-539 176 178 178 314 320 320 F F F <--- 27 540-559 176 176 176 316 316 316 F F F 28 560-579 176 176 176 314 314 314 F F F 29 580-599 176 176 176 314 314 314 F F F Max gen: 178, Max flush: 322 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 178 178 178 320 320 320 F F F 1 020-039 177 177 177 318 318 318 F F F 2 040-059 177 177 177 318 318 318 F F F 3 060-079 171 171 171 307 307 307 F F F 4 080-099 172 172 172 309 309 309 F F F 5 100-119 176 176 176 316 316 316 F F F 6 120-139 178 178 178 320 320 320 F F F 7 140-159 179 179 179 323 323 323 F F F 8 160-179 167 167 167 300 300 300 F F F 9 180-199 178 178 178 319 319 319 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 178 178 178 322 322 322 F F F 12 240-259 179 179 179 323 323 323 F F F 13 260-279 178 178 178 322 322 322 F F F 14 280-299 176 176 176 314 314 314 F F F 15 300-319 176 176 176 316 316 316 F F F 16 320-339 174 174 174 311 311 311 F F F 17 340-359 179 179 179 323 323 323 F F F 18 360-379 179 179 179 323 323 323 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 F F F 21 420-439 178 178 178 320 320 320 F F F 22 440-459 178 178 178 320 320 320 F F F 23 460-479 178 178 178 320 320 320 F F F 24 480-499 178 178 178 320 320 320 F F F 25 500-519 178 178 178 320 320 320 F F F 26 520-539 178 178 178 320 320 320 F F F 27 540-559 176 176 176 316 316 316 F F F 28 560-579 176 176 176 314 314 314 F F F 29 580-599 176 176 176 314 314 314 F F F Max gen: 179, Max flush: 323 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.662139173Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.693224027Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.694178687Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.69426647Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.694292302Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.694327782Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915} {"msg":"Crucible 36aaf3a9-bca4-4014-821d-edd7ca087a69 has session id: 4ae6bbc0-58b4-4cea-aa38-04978f9b25a0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.695760559Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"Upstairs opts: Upstairs UUID: 36aaf3a9-bca4-4014-821d-edd7ca087a69, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.695838613Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"Crucible stats registered with UUID: 36aaf3a9-bca4-4014-821d-edd7ca087a69","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.695885606Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:00.695930194Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.696457949Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.696538469Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.696576434Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.69705434Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.69710461Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915} {"msg":"36aaf3a9-bca4-4014-821d-edd7ca087a69 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.697211461Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.697255318Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"client":"0","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.697293724Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"client":"1","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.697331819Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"client":"2","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.69737257Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.69741297Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.697449723Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:44420)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.697678606Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"2","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:00.697734146Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:00.697770638Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:00.697806078Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:33454)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.697831609Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"0","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.698078568Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.698119639Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.699658326Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"client":"0","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.699795156Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"client":"2","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.700570882Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:00.700679187Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.699338065Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:57963)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.699551358Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"io task","client":"1","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.702019925Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"client":"1","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"[0]R flush_numbers[0..12]: [320, 318, 318, 307, 309, 316, 320, 323, 300, 319, 317, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703222927Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"downstairs","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"[0]R generation[0..12]: [178, 177, 177, 171, 172, 176, 178, 179, 167, 178, 176, 178]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703322825Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"downstairs","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703370499Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"downstairs","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"[1]R flush_numbers[0..12]: [320, 318, 318, 307, 309, 316, 320, 323, 300, 319, 317, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.70341152Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"downstairs","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"[1]R generation[0..12]: [178, 177, 177, 171, 172, 176, 178, 179, 167, 178, 176, 178]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703453554Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"downstairs","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703497771Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"downstairs","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"[2]R flush_numbers[0..12]: [320, 318, 318, 307, 309, 316, 320, 323, 300, 319, 317, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703536778Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"downstairs","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"[2]R generation[0..12]: [178, 177, 177, 171, 172, 176, 178, 179, 167, 178, 176, 178]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703589222Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"downstairs","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703637658Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"downstairs","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703676415Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"downstairs","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703719099Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"downstairs","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703752996Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"downstairs","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703788527Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"":"downstairs","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703829027Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703866211Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"client":"0","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name"Wait for a query_work_queue command to finish before sending IO :"crucible","level":30,"time":"2026-03-04T01:17:10.703923414Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"client":"1","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.703963183Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"client":"2","":"downstairs","session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.70403274Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.704076477Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"36aaf3a9-bca4-4014-821d-edd7ca087a69 is now active with session: 4ae6bbc0-58b4-4cea-aa38-04978f9b25a0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.704122067Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.704160764Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915,"session_id":"4ae6bbc0-58b4-4cea-aa38-04978f9b25a0"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.70419937Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915} {"msg":"Activated sub_volume 36aaf3a9-bca4-4014-821d-edd7ca087a69","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:10.704235712Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 510 len 16384 data: 14 13 13 12 02/30 Write block 559 len 28672 data: 12 13 12 15 17 18 15 03/30 Write block 535 len 28672 data: 15 17 16 17 17 15 15 04/30 Write block 136 len 20480 data: 15 15 15 20 15 05/30 Flush 06/30 Write block 347 len 28672 data: 11 10 11 11 9 8 8 07/30 Write block 1 len 24576 data: 4 9 13 16 18 18 08/30 Read block 501 len 24576 09/30 Write block 496 len 20480 data: 14 14 13 13 9 10/30 Flush 11/30 Write block 275 len 32768 data: 17 16 18 18 17 16 14 14 12/30 Read block 259 len 28672 13/30 Write block 313 len 32768 data: 12 10 13 15 12 12 13 18 14/30 Write block 289 len 20480 data: 14 14 19 18 13 15/30 Write block 505 len 28672 data: 11 9 10 10 12 15 14 16/30 Read block 104 len 8192 17/30 Read block 210 len 36864 18/30 Read block 388 len 40960 19/30 Write block 522 len 40960 data: 16 17 16 15 13 13 15 17 16 16 20/30 Read block 419 len 16384 21/30 Read block 481 len 8192 22/30 Read block 533 len 24576 23/30 Read block 44 len 32768 24/30 Write block 302 len 36864 data: 13 15 16 16 16 17 18 19 17 25/30 Read block 305 len 8192 26/30 Read block 571 len 12288 27/30 Write block 71 len 36864 data: 12 13 13 11 12 9 7 6 7 28/30 Read block 424 len 4096 29/30 Write block 410 len 4096 data: 12 30/30 Read block 63 len 24576 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Sent Done false 1001 Acked Write 7 Done Sent Done false 1002 Acked Write 7 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) None Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:10.74308586Z","hostname":"bmat-EVT22200007-0000ead9","pid":1915} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 178 180 325 320 325 F F F <--- 1 020-039 177 177 177 318 318 318 F F F 2 040-059 177 177 177 318 318 318 F F F 3 060-079 171 171 171 307 307 307 T F T <--- 4 080-099 172 172 172 309 309 309 F F F 5 100-119 176 176 176 316 316 316 F F F 6 120-139 180 178 180 324 320 324 F F F <--- 7 140-159 180 179 180 324 323 324 F F F <--- 8 160-179 167 167 167 300 300 300 F F F 9 180-199 178 178 178 319 319 319 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 178 178 178 322 322 322 F F F 12 240-259 179 179 179 323 323 323 F F F 13 260-279 178 178 178 322 322 322 T F T <--- 14 280-299 176 176 176 314 314 314 T F T <--- 15 300-319 176 176 176 316 316 316 T F T <--- 16 320-339 174 174 174 311 311 311 T F T <--- 17 340-359 180 179 180 325 323 325 F F F <--- 18 360-379 179 179 179 323 323 323 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 167 167 167 300 300 300 T F T <--- 21 420-439 178 178 178 320 320 320 F F F 22 440-459 178 178 178 320 320 320 F F F 23 460-479 178 178 178 320 320 320 F F F 24 480-499 180 178 180 325 320 325 F F F <--- 25 500-519 180 178 180 325 320 325 T F T <--- 26 520-539 180 178 180 324 320 324 T F T <--- 27 540-559 180 176 180 324 316 324 F F F <--- 28 560-579 180 176 180 324 314 324 F F F <--- 29 580-599 176 176 176 314 314 314 F F F Max gen: 180, Max flush: 325 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 325 325 325 F F F 1 020-039 177 177 177 318 318 318 F F F 2 040-059 177 177 177 318 318 318 F F F 3 060-079 181 181 181 326 326 326 F F F 4 080-099 172 172 172 309 309 309 F F F 5 100-119 176 176 176 316 316 316 F F F 6 120-139 180 180 180 324 324 324 F F F 7 140-159 180 180 180 324 324 324 F F F 8 160-179 167 167 167 300 300 300 F F F 9 180-199 178 178 178 319 319 319 F F F 10 200-219 176 176 176 317 317 317 F F F 11 220-239 178 178 178 322 322 322 F F F 12 240-259 179 179 179 323 323 323 F F F 13 260-279 181 181 181 326 326 326 F F F 14 280-299 181 181 181 326 326 326 F F F 15 300-319 181 181 181 326 326 326 F F F 16 320-339 181 181 181 326 326 326 F F F 17 340-359 180 180 180 325 325 325 F F F 18 360-379 179 179 179 323 323 323 F F F 19 380-399 153 153 153 273 273 273 F F F 20 400-419 181 181 181 326 326 326 F F F 21 420-439 178 178 178 320 320 320 F F F 22 440-459 178 178 178 320 320 320 F F F 23 460-479 178 178 178 320 320 320 F F F 24 480-499 180 180 180 325 325 325 F F F 25 500-519 181 181 181 326 326 326 F F F 26 520-539 181 181 181 326 326 326 F F F 27 540-559 180 180 180 324 324 324 F F F 28 560-579 180 180 180 324 324 324 F F F 29 580-599 176 176 176 314 314 314 F F F Max gen: 181, Max flush: 326 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.447108402Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:15.469123138Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.469234619Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.491853661Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.492419562Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.49249443Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.492522255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.492560982Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923} {"msg":"Crucible 12143a4d-80ae-4cc4-8c9c-05d0c1c7f1b5 has session id: da2f9dcf-08d6-445e-b3f9-52ec683db6e9","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.493731449Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"Upstairs opts: Upstairs UUID: 12143a4d-80ae-4cc4-8c9c-05d0c1c7f1b5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.493854062Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"Crucible stats registered with UUID: 12143a4d-80ae-4cc4-8c9c-05d0c1c7f1b5","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.493890404Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:15.493921065Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.494313471Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.49439865Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.494436926Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.494465172Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.494548677Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923} {"msg":"12143a4d-80ae-4cc4-8c9c-05d0c1c7f1b5 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.494745797Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.494776117Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"client":"0","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.494803211Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"client":"1","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.494829884Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"client":"2","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.494923489Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.494956034Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.494983678Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:57499)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.495553016Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"io task","client":"1","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:40174)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.495602403Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"io task","client":"2","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:63128)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.495813391Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"io task","client":"0","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.497279694Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"client":"1","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.497335986Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"client":"0","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.497513768Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"client":"2","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.498005802Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.498221409Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"[0]R flush_numbers[0..12]: [325, 318, 318, 326, 309, 316, 324, 324, 300, 319, 317, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.498404201Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"downstairs","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"[0]R generation[0..12]: [180, 177, 177, 181, 172, 176, 180, 180, 167, 178, 176, 178]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.498471454Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"downstairs","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.498526994Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"downstairs","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"[1]R flush_numbers[0..12]: [325, 318, 318, 326, 309, 316, 324, 324, 300, 319, 317, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.498576923Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"downstairs","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"[1]R generation[0..12]: [180, 177, 177, 181, 172, 176, 180, 180, 167, 178, 176, 178]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.49862601Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"downstairs","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.498681439Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"[2]R flush_numbers[0..12]: [325, 318, 318, 326, 309, 316, 324, 324, 300, 319, 317, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.498745697Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"downstairs","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"[2]R generation[0..12]: [180, 177, 177, 181, 172, 176, 180, 180, 167, 178, 176, 178]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.498795626Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"downstairs","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.498846416Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"downstairs","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.498887738Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"downstairs","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.498936705Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"downstairs","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"Next flush: 327","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.498991684Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"downstairs","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.499038727Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"":"downstairs","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.499084598Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.49912609Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"client":"0","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.499172141Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"client":"1","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.499215296Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"client":"2","":"downstairs","session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.499259043Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.499293451Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"12143a4d-80ae-4cc4-8c9c-05d0c1c7f1b5 is now active with session: da2f9dcf-08d6-445e-b3f9-52ec683db6e9","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.49932839Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.499360714Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923,"session_id":"da2f9dcf-08d6-445e-b3f9-52ec683db6e9"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.499390583Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923} {"msg":"Activated sub_volume 12143a4d-80ae-4cc4-8c9c-05d0c1c7f1b5","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:15.49941895Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 489 len 40960 data: 12 13 12 12 10 13 13 15 15 14 02/30 Read block 165 len 24576 03/30 Flush 04/30 Write block 392 len 8192 data: 17 15 05/30 Read block 228 len 20480 06/30 Write block 152 len 28672 data: 11 13 14 13 13 15 16 07/30 Write block 302 len 12288 data: 14 16 17 08/30 Read block 413 len 36864 09/30 Read block 410 len 32768 10/30 Read block 340 len 28672 11/30 Write block 566 len 8192 data: 15 17 12/30 Write block 309 len 28672 data: 20 18 16 13 13 11 14 13/30 Read block 354 len 20480 14/30 Write block 357 len 8192 data: 16 21 15/30 Read block 365 len 20480 16/30 Flush 17/30 Write block 456 len 16384 data: 12 15 18 18 18/30 Flush 19/30 Write block 397 len 16384 data: 11 9 11 15 20/30 Flush 21/30 Write block 238 len 4096 data: 20 22/30 Write block 584 len 36864 data: 19 16 15 8 9 12 13 10 12 23/30 Write block 289 len 12288 data: 15 15 20 24/30 Read block 283 len 36864 25/30 Flush 26/30 Write block 121 len 20480 data: 14 12 10 11 11 27/30 Read block 568 len 16384 28/30 Write block 208 len 20480 data: 20 20 18 13 14 29/30 Write block 278 len 4096 data: 19 30/30 Read block 174 len 28672 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Write 7 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Write 5 Sent Done Sent false 1028 Acked Write 1 Sent Done Sent false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 3 31 Done 2 30 27 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:15.532417643Z","hostname":"bmat-EVT22200007-0000ead9","pid":1923} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 325 325 325 F F F 1 020-039 177 177 177 318 318 318 F F F 2 040-059 177 177 177 318 318 318 F F F 3 060-079 181 181 181 326 326 326 F F F 4 080-099 172 172 172 309 309 309 F F F 5 100-119 176 176 176 316 316 316 F F F 6 120-139 180 180 180 324 324 324 F T T <--- 7 140-159 180 182 182 324 328 328 F F F <--- 8 160-179 167 167 167 300 300 300 F F F 9 180-199 178 178 178 319 319 319 F F F 10 200-219 176 176 176 317 317 317 F T T <--- 11 220-239 178 182 182 322 331 331 F F F <--- 12 240-259 179 179 179 323 323 323 F F F 13 260-279 181 181 181 326 326 326 F T T <--- 14 280-299 181 182 182 326 331 331 F F F <--- 15 300-319 181 182 182 326 328 328 F F F <--- 16 320-339 181 181 181 326 326 326 F F F 17 340-359 180 182 182 325 328 328 F F F <--- 18 360-379 179 179 179 323 323 323 F F F 19 380-399 153 182 182 273 330 330 F F F <--- 20 400-419 181 182 182 326 330 330 F F F <--- 21 420-439 178 178 178 320 320 320 F F F 22 440-459 178 182 182 320 329 329 F F F <--- 23 460-479 178 178 178 320 320 320 F F F 24 480-499 180 182 182 325 327 327 T F F <--- 25 500-519 181 181 181 326 326 326 F F F 26 520-539 181 181 181 326 326 326 F F F 27 540-559 180 180 180 324 324 324 F F F 28 560-579 180 182 182 324 328 328 F F F <--- 29 580-599 176 182 182 314 331 331 F F F <--- Max gen: 182, Max flush: 331 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 325 325 325 F F F 1 020-039 177 177 177 318 318 318 F F F 2 040-059 177 177 177 318 318 318 F F F 3 060-079 181 181 181 326 326 326 F F F 4 080-099 172 172 172 309 309 309 F F F 5 100-119 176 176 176 316 316 316 F F F 6 120-139 183 183 183 332 332 332 F F F 7 140-159 182 182 182 328 328 328 F F F 8 160-179 167 167 167 300 300 300 F F F 9 180-199 178 178 178 319 319 319 F F F 10 200-219 183 183 183 332 332 332 F F F 11 220-239 182 182 182 331 331 331 F F F 12 240-259 179 179 179 323 323 323 F F F 13 260-279 183 183 183 332 332 332 F F F 14 280-299 182 182 182 331 331 331 F F F 15 300-319 182 182 182 328 328 328 F F F 16 320-339 181 181 181 326 326 326 F F F 17 340-359 182 182 182 328 328 328 F F F 18 360-379 179 179 179 323 323 323 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 182 182 182 330 330 330 F F F 21 420-439 178 178 178 320 320 320 F F F 22 440-459 182 182 182 329 329 329 F F F 23 460-479 178 178 178 320 320 320 F F F 24 480-499 182 182 182 327 327 327 F F F 25 500-519 181 181 181 326 326 326 F F F 26 520-539 181 181 181 326 326 326 F F F 27 540-559 180 180 180 324 324 324 F F F 28 560-579 182 182 182 328 328 328 F F F 29 580-599 182 182 182 331 331 331 F F F Max gen: 183, Max flush: 332 Begin loop 92 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.237876288Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.269017353Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.269929449Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.270077612Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.270113233Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.270166989Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931} {"msg":"Crucible cc4f1c11-10aa-4d27-ba80-8889e8680da0 has session id: 06563b40-bdde-4662-99b6-38590696725e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.271448256Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"Upstairs opts: Upstairs UUID: cc4f1c11-10aa-4d27-ba80-8889e8680da0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.27153627Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"Crucible stats registered with UUID: cc4f1c11-10aa-4d27-ba80-8889e8680da0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.271586259Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:20.271624976Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.272168392Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.272280303Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.272318198Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.272559215Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.272605517Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931} {"msg":"cc4f1c11-10aa-4d27-ba80-8889e8680da0 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.272832607Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.272882596Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"client":"0","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.27293006Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"client":"1","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.272974498Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"client":"2","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.273014807Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.273058804Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.273117841Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43665)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.273340222Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"io task","client":"1","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"ds_connection connected from Ok(127.0.0.1:53308)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.273641939Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"io task","client":"2","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45078)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.273702529Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"io task","client":"0","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.275604124Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"client":"0","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.275669784Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"client":"1","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.275708901Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"client":"2","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.276529646Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.276616899Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"[0]R flush_numbers[0..12]: [325, 318, 318, 326, 309, 316, 332, 328, 300, 319, 332, 331]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.276811134Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"downstairs","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"[0]R generation[0..12]: [180, 177, 177, 181, 172, 176, 183, 182, 167, 178, 183, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.276865541Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"downstairs","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.27690538Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"downstairs","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"[1]R flush_numbers[0..12]: [325, 318, 318, 326, 309, 316, 332, 328, 300, 319, 332, 331]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.276949257Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"downstairs","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"[1]R generation[0..12]: [180, 177, 177, 181, 172, 176, 183, 182, 167, 178, 183, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.276987302Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"downstairs","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277036309Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"downstairs","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"[2]R flush_numbers[0..12]: [325, 318, 318, 326, 309, 316, 332, 328, 300, 319, 332, 331]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277074575Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"downstairs","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"[2]R generation[0..12]: [180, 177, 177, 181, 172, 176, 183, 182, 167, 178, 183, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277111699Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"downstairs","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277151297Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"downstairs","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"Max found gen is 184","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.27719846Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"downstairs","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277233049Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"downstairs","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277264321Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"downstairs","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277303929Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"":"downstairs","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277350501Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277388416Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"client":"0","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277431873Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"client":"1","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.27747593Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"client":"2","":"downstairs","session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277513725Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277545107Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"cc4f1c11-10aa-4d27-ba80-8889e8680da0 is now active with session: 06563b40-bdde-4662-99b6-38590696725e","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277580747Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-04T01:17:20.277615777Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931,"session_id":"06563b40-bdde-4662-99b6-38590696725e"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277654022Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931} {"msg":"Activated sub_volume cc4f1c11-10aa-4d27-ba80-8889e8680da0","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:20.277681387Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 385 len 4096 data: 13 02/30 Write block 477 len 8192 data: 11 10 03/30 Write block 535 len 20480 data: 16 18 17 18 18 04/30 Write block 78 len 12288 data: 7 8 8 05/30 Write block 403 len 36864 data: 15 17 16 17 16 15 12 13 11 06/30 Write block 376 len 36864 data: 10 11 11 12 11 10 10 9 10 07/30 Read block 315 len 32768 08/30 Read block 528 len 16384 09/30 Read block 100 len 32768 10/30 Write block 149 len 12288 data: 10 10 13 11/30 Read block 348 len 12288 12/30 Read block 247 len 8192 13/30 Read block 401 len 28672 14/30 Read block 525 len 24576 15/30 Write block 288 len 20480 data: 15 16 16 21 19 16/30 Read block 487 len 12288 17/30 Write block 95 len 32768 data: 15 18 20 17 16 16 8 9 18/30 Read block 543 len 32768 19/30 Write block 82 len 24576 data: 14 13 16 16 16 16 20/30 Read block 391 len 40960 21/30 Write block 578 len 24576 data: 19 18 20 18 17 17 22/30 Write block 83 len 32768 data: 14 17 17 17 17 14 15 12 23/30 Read block 519 len 28672 24/30 Write block 359 len 12288 data: 20 17 18 25/30 Read block 507 len 24576 26/30 Read block 58 len 32768 27/30 Read block 544 len 4096 28/30 Read block 591 len 16384 29/30 Write block 551 len 12288 data: 13 16 16 30/30 Read block 442 len 12288 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Read 8 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Read 6 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 3 Done Sent Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 18 1 19 Done 30 12 29 71 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:20.313638336Z","hostname":"bmat-EVT22200007-0000ead9","pid":1931} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 325 325 325 F F F 1 020-039 177 177 177 318 318 318 F F F 2 040-059 177 177 177 318 318 318 F F F 3 060-079 181 181 181 326 326 326 T T T 4 080-099 172 172 172 309 309 309 T T T 5 100-119 176 176 176 316 316 316 T F T <--- 6 120-139 183 183 183 332 332 332 F F F 7 140-159 182 182 182 328 328 328 T T T 8 160-179 167 167 167 300 300 300 F F F 9 180-199 178 178 178 319 319 319 F F F 10 200-219 183 183 183 332 332 332 F F F 11 220-239 182 182 182 331 331 331 F F F 12 240-259 179 179 179 323 323 323 F F F 13 260-279 183 183 183 332 332 332 F F F 14 280-299 182 182 182 331 331 331 T F T <--- 15 300-319 182 182 182 328 328 328 F F F 16 320-339 181 181 181 326 326 326 F F F 17 340-359 182 182 182 328 328 328 T F T <--- 18 360-379 179 179 179 323 323 323 T T T 19 380-399 182 182 182 330 330 330 T T T 20 400-419 182 182 182 330 330 330 T T T 21 420-439 178 178 178 320 320 320 F F F 22 440-459 182 182 182 329 329 329 F F F 23 460-479 178 178 178 320 320 320 T T T 24 480-499 182 182 182 327 327 327 F F F 25 500-519 181 181 181 326 326 326 F F F 26 520-539 181 181 181 326 326 326 T T T 27 540-559 180 180 180 324 324 324 T F T <--- 28 560-579 182 182 182 328 328 328 T F T <--- 29 580-599 182 182 182 331 331 331 T F T <--- Max gen: 183, Max flush: 332 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 325 325 325 F F F 1 020-039 177 177 177 318 318 318 F F F 2 040-059 177 177 177 318 318 318 F F F 3 060-079 184 184 184 333 333 333 F F F 4 080-099 184 184 184 333 333 333 F F F 5 100-119 184 184 184 333 333 333 F F F 6 120-139 183 183 183 332 332 332 F F F 7 140-159 184 184 184 333 333 333 F F F 8 160-179 167 167 167 300 300 300 F F F 9 180-199 178 178 178 319 319 319 F F F 10 200-219 183 183 183 332 332 332 F F F 11 220-239 182 182 182 331 331 331 F F F 12 240-259 179 179 179 323 323 323 F F F 13 260-279 183 183 183 332 332 332 F F F 14 280-299 184 184 184 333 333 333 F F F 15 300-319 182 182 182 328 328 328 F F F 16 320-339 181 181 181 326 326 326 F F F 17 340-359 184 184 184 333 333 333 F F F 18 360-379 184 184 184 333 333 333 F F F 19 380-399 184 184 184 333 333 333 F F F 20 400-419 184 184 184 333 333 333 F F F 21 420-439 178 178 178 320 320 320 F F F 22 440-459 182 182 182 329 329 329 F F F 23 460-479 184 184 184 333 333 333 F F F 24 480-499 182 182 182 327 327 327 F F F 25 500-519 181 181 181 326 326 326 F F F 26 520-539 184 184 184 333 333 333 F F F 27 540-559 184 184 184 333 333 333 F F F 28 560-579 184 184 184 333 333 333 F F F 29 580-599 184 184 184 333 333 333 F F F Max gen: 184, Max flush: 333 Begin loop 93 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:24.98416957Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:25.009744595Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.009832279Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.033572611Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.034078834Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.034174914Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.034218861Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.034275573Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939} {"msg":"Crucible 921c133d-ba5f-4d8e-8050-ef0f4d4aaaeb has session id: 4c48ba06-326c-4269-93dc-af5db8321693","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.035277527Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"Upstairs opts: Upstairs UUID: 921c133d-ba5f-4d8e-8050-ef0f4d4aaaeb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.035353848Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"Crucible stats registered with UUID: 921c133d-ba5f-4d8e-8050-ef0f4d4aaaeb","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.035390461Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:25.03542046Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.035818097Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.03589532Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.035927754Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.036213991Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.036244652Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939} {"msg":"921c133d-ba5f-4d8e-8050-ef0f4d4aaaeb active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.036370571Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.036399829Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"client":"0","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.036427153Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"client":"1","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.036454247Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"client":"2","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.036481952Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.036512322Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.03662266Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"ds_connection connected from Ok(127.0.0.1:53443)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.037055577Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"io task","client":"2","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"ds_connection connected from Ok(127.0.0.1:51882)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.037134884Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"io task","client":"1","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"ds_connection connected from Ok(127.0.0.1:40150)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.037165685Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"io task","client":"0","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.039254551Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"client":"0","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.039511179Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"client":"2","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.039574114Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"client":"1","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040205774Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040403065Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"[0]R flush_numbers[0..12]: [325, 318, 318, 333, 333, 333, 332, 333, 300, 319, 332, 331]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040512602Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"downstairs","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"[0]R generation[0..12]: [180, 177, 177, 184, 184, 184, 183, 184, 167, 178, 183, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040550417Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"downstairs","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040582881Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"downstairs","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"[1]R flush_numbers[0..12]: [325, 318, 318, 333, 333, 333, 332, 333, 300, 319, 332, 331]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040613852Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"downstairs","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"[1]R generation[0..12]: [180, 177, 177, 184, 184, 184, 183, 184, 167, 178, 183, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040644543Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"downstairs","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040676396Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"downstairs","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"[2]R flush_numbers[0..12]: [325, 318, 318, 333, 333, 333, 332, 333, 300, 319, 332, 331]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040707077Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"downstairs","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"[2]R generation[0..12]: [180, 177, 177, 184, 184, 184, 183, 184, 167, 178, 183, 182]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040737527Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"downstairs","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.04076955Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"downstairs","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"Max found gen is 185","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040795843Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"downstairs","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"Generation requested: 186 >= found:185","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040831804Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"downstairs","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040857555Z","hostname":"Wait for a query_work_queue command to finish before sending IO bmat-EVT22200007-0000ead9","pid":1939,"":"downstairs","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040899558Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"":"downstairs","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040926672Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.04095132Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"client":"0","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.040978735Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"client":"1","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.041005548Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"client":"2","":"downstairs","session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.041033663Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.041059254Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"921c133d-ba5f-4d8e-8050-ef0f4d4aaaeb is now active with session: 4c48ba06-326c-4269-93dc-af5db8321693","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.041088031Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.041112139Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939,"session_id":"4c48ba06-326c-4269-93dc-af5db8321693"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.041137669Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939} {"msg":"Activated sub_volume 921c133d-ba5f-4d8e-8050-ef0f4d4aaaeb","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:25.041159533Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 473 len 24576 data: 12 11 14 12 12 11 02/30 Write block 373 len 12288 data: 11 12 8 03/30 Write block 528 len 20480 data: 16 18 17 17 15 04/30 Write block 185 len 4096 data: 14 05/30 Write block 345 len 4096 data: 11 06/30 Write block 238 len 36864 data: 21 21 20 17 13 12 10 10 10 07/30 Write block 135 len 32768 data: 15 16 16 16 21 16 12 10 08/30 Read block 268 len 4096 09/30 Write block 164 len 12288 data: 17 16 12 10/30 Read block 95 len 8192 11/30 Read block 596 len 12288 12/30 Write block 95 len 36864 data: 16 19 21 18 17 17 9 10 5 13/30 Write block 514 len 24576 data: 14 12 11 10 11 12 14/30 Read block 80 len 24576 15/30 Read block 263 len 4096 16/30 Read block 400 len 20480 17/30 Write block 378 len 28672 data: 12 13 12 11 11 10 11 18/30 Read block 443 len 28672 19/30 Read block 171 len 32768 20/30 Write block 276 len 16384 data: 17 19 20 18 21/30 Read block 210 len 32768 22/30 Read block 273 len 24576 23/30 Write block 312 len 40960 data: 14 14 12 15 16 13 13 14 19 16 24/30 Write block 488 len 28672 data: 14 13 14 13 13 11 14 25/30 Read block 494 len 8192 26/30 Write block 407 len 8192 data: 17 16 27/30 Write block 138 len 24576 data: 17 22 17 13 11 12 28/30 Read block 22 len 8192 29/30 Read block 565 len 24576 30/30 Write block 207 len 4096 data: 20 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Sent Done Done false 1001 Acked Write 3 Sent Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Write 8 Sent Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:25.070581855Z","hostname":"bmat-EVT22200007-0000ead9","pid":1939} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 325 325 325 F F F 1 020-039 177 177 177 318 318 318 F F F 2 040-059 177 177 177 318 318 318 F F F 3 060-079 184 184 184 333 333 333 F F F 4 080-099 184 184 184 333 333 333 F T T <--- 5 100-119 184 184 184 333 333 333 F T T <--- 6 120-139 183 183 183 332 332 332 F T T <--- 7 140-159 184 184 184 333 333 333 F T T <--- 8 160-179 167 167 167 300 300 300 F T T <--- 9 180-199 178 178 178 319 319 319 F T T <--- 10 200-219 183 183 183 332 332 332 F T T <--- 11 220-239 182 182 182 331 331 331 F T T <--- 12 240-259 179 179 179 323 323 323 F T T <--- 13 260-279 183 183 183 332 332 332 F T T <--- 14 280-299 184 184 184 333 333 333 F F F 15 300-319 182 182 182 328 328 328 F T T <--- 16 320-339 181 181 181 326 326 326 F T T <--- 17 340-359 184 184 184 333 333 333 F T T <--- 18 360-379 184 184 184 333 333 333 F T T <--- 19 380-399 184 184 184 333 333 333 F T T <--- 20 400-419 184 184 184 333 333 333 F T T <--- 21 420-439 178 178 178 320 320 320 F F F 22 440-459 182 182 182 329 329 329 F F F 23 460-479 184 184 184 333 333 333 F T T <--- 24 480-499 182 182 182 327 327 327 F T T <--- 25 500-519 181 181 181 326 326 326 F T T <--- 26 520-539 184 184 184 333 333 333 F T T <--- 27 540-559 184 184 184 333 333 333 F F F 28 560-579 184 184 184 333 333 333 F F F 29 580-599 184 184 184 333 333 333 F F F Max gen: 184, Max flush: 333 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 325 325 325 F F F 1 020-039 177 177 177 318 318 318 F F F 2 040-059 177 177 177 318 318 318 F F F 3 060-079 184 184 184 333 333 333 F F F 4 080-099 185 185 185 334 334 334 F F F 5 100-119 185 185 185 334 334 334 F F F 6 120-139 185 185 185 334 334 334 F F F 7 140-159 185 185 185 334 334 334 F F F 8 160-179 185 185 185 334 334 334 F F F 9 180-199 185 185 185 334 334 334 F F F 10 200-219 185 185 185 334 334 334 F F F 11 220-239 185 185 185 334 334 334 F F F 12 240-259 185 185 185 334 334 334 F F F 13 260-279 185 185 185 334 334 334 F F F 14 280-299 184 184 184 333 333 333 F F F 15 300-319 185 185 185 334 334 334 F F F 16 320-339 185 185 185 334 334 334 F F F 17 340-359 185 185 185 334 334 334 F F F 18 360-379 185 185 185 334 334 334 F F F 19 380-399 185 185 185 334 334 334 F F F 20 400-419 185 185 185 334 334 334 F F F 21 420-439 178 178 178 320 320 320 F F F 22 440-459 182 182 182 329 329 329 F F F 23 460-479 185 185 185 334 334 334 F F F 24 480-499 185 185 185 334 334 334 F F F 25 500-519 185 185 185 334 334 334 F F F 26 520-539 185 185 185 334 334 334 F F F 27 540-559 184 184 184 333 333 333 F F F 28 560-579 184 184 184 333 333 333 F F F 29 580-599 184 184 184 333 333 333 F F F Max gen: 185, Max flush: 334 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.790815473Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.821451578Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.822048239Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.822120913Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.822152886Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.822188858Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947} {"msg":"Crucible 3f52f831-80d2-4311-bfa0-0a2ca389b6b7 has session id: 181b6e01-5eef-41e2-b5ef-12eb16da1916","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.823312212Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"Upstairs opts: Upstairs UUID: 3f52f831-80d2-4311-bfa0-0a2ca389b6b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.823373964Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"Crucible stats registered with UUID: 3f52f831-80d2-4311-bfa0-0a2ca389b6b7","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.823403422Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:29.823428793Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.823919304Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.823975144Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.824003601Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.824379695Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.824412029Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947} {"msg":"3f52f831-80d2-4311-bfa0-0a2ca389b6b7 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.824580994Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.824610031Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"client":"0","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.824637155Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"client":"1","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.82467018Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"client":"2","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.824697314Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.82472557Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.824759557Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"ds_connection connected from Ok(127.0.0.1:50629)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.825375397Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"io task","client":"0","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"ds_connection connected from Ok(127.0.0.1:53912)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.825463271Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"io task","client":"1","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"ds_connection connected from Ok(127.0.0.1:40239)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.825661222Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"io task","client":"2","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.826654108Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"client":"1","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.826768485Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"client":"0","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.826929123Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"client":"2","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.827577156Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.827725219Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"[0]R flush_numbers[0..12]: [325, 318, 318, 333, 334, 334, 334, 334, 334, 334, 334, 334]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.827785528Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"downstairs","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"[0]R generation[0..12]: [180, 177, 177, 184, 185, 185, 185, 185, 185, 185, 185, 185]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.827842511Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"downstairs","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.827900336Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"downstairs","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"[1]R flush_numbers[0..12]: [325, 318, 318, 333, 334, 334, 334, 334, 334, 334, 334, 334]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.827965104Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"downstairs","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"[1]R generation[0..12]: [180, 177, 177, 184, 185, 185, 185, 185, 185, 185, 185, 185]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828020854Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"downstairs","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828078839Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"downstairs","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"[2]R flush_numbers[0..12]: [325, 318, 318, 333, 334, 334, 334, 334, 334, 334, 334, 334]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828134189Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"downstairs","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"[2]R generation[0..12]: [180, 177, 177, 184, 185, 185, 185, 185, 185, 185, 185, 185]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828188516Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"downstairs","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828245379Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"downstairs","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828287572Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"downstairs","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"Generation requested: 188 >= found:186","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828329145Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"downstairs","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"Wait for a query_work_queue command to finish before sending IO msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828381679Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"downstairs","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828429012Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"":"downstairs","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828482558Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828528539Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"client":"0","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828576855Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"client":"1","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828625231Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"client":"2","":"downstairs","session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828674829Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.82872583Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"3f52f831-80d2-4311-bfa0-0a2ca389b6b7 is now active with session: 181b6e01-5eef-41e2-b5ef-12eb16da1916","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828778023Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828819576Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947,"session_id":"181b6e01-5eef-41e2-b5ef-12eb16da1916"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828856609Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947} {"msg":"Activated sub_volume 3f52f831-80d2-4311-bfa0-0a2ca389b6b7","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:29.828895566Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"bmat-EVT22200007-0000ead9","pid":1947} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 195 len 20480 data: 14 16 16 17 17 02/30 Read block 383 len 40960 03/30 Read block 551 len 24576 04/30 Write block 20 len 40960 data: 17 16 17 13 12 9 7 11 12 13 05/30 Write block 62 len 36864 data: 9 10 13 11 10 10 13 15 14 06/30 Read block 89 len 36864 07/30 Flush 08/30 Flush 09/30 Write block 96 len 40960 data: 20 22 19 18 18 10 11 6 8 9 10/30 Write block 300 len 8192 data: 11 10 11/30 Read block 312 len 36864 12/30 Write block 513 len 36864 data: 13 15 13 12 11 12 13 10 12 13/30 Read block 8 len 4096 14/30 Write block 167 len 36864 data: 12 14 11 10 11 10 10 13 13 15/30 Write block 197 len 12288 data: 17 18 18 16/30 Write block 511 len 8192 data: 15 14 17/30 Write block 393 len 20480 data: 16 12 10 12 12 18/30 Write block 574 len 32768 data: 13 16 14 16 20 19 21 19 19/30 Write block 526 len 20480 data: 14 14 17 19 18 20/30 Read block 468 len 24576 21/30 Write block 276 len 8192 data: 18 20 22/30 Write block 65 len 8192 data: 12 11 23/30 Write block 524 len 24576 data: 17 16 15 15 18 20 24/30 Write block 252 len 28672 data: 8 10 12 12 11 15 15 25/30 Read block 61 len 32768 26/30 Read block 121 len 4096 27/30 Read block 30 len 36864 28/30 Write block 276 len 24576 data: 19 21 21 19 17 15 29/30 Read block 241 len 36864 30/30 Read block 527 len 12288 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Sent false 1001 Acked Read 10 Done Done Sent false 1002 Acked Read 6 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:29.862130416Z","hostname":"bmat-EVT22200007-0000ead9","pid":1947} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 325 325 325 F F F 1 020-039 188 188 177 335 335 318 F F F <--- 2 040-059 177 177 177 318 318 318 F F F 3 060-079 188 188 184 335 335 333 T T F <--- 4 080-099 185 185 185 334 334 334 T T F <--- 5 100-119 185 185 185 334 334 334 T T F <--- 6 120-139 185 185 185 334 334 334 F F F 7 140-159 185 185 185 334 334 334 F F F 8 160-179 185 185 185 334 334 334 T T F <--- 9 180-199 188 188 185 335 335 334 T T F <--- 10 200-219 185 185 185 334 334 334 F F F 11 220-239 185 185 185 334 334 334 F F F 12 240-259 185 185 185 334 334 334 T T F <--- 13 260-279 185 185 185 334 334 334 T T F <--- 14 280-299 184 184 184 333 333 333 T T F <--- 15 300-319 185 185 185 334 334 334 T T F <--- 16 320-339 185 185 185 334 334 334 F F F 17 340-359 185 185 185 334 334 334 F F F 18 360-379 185 185 185 334 334 334 F F F 19 380-399 185 185 185 334 334 334 T T F <--- 20 400-419 185 185 185 334 334 334 F F F 21 420-439 178 178 178 320 320 320 F F F 22 440-459 182 182 182 329 329 329 F F F 23 460-479 185 185 185 334 334 334 F F F 24 480-499 185 185 185 334 334 334 F F F 25 500-519 185 185 185 334 334 334 T T F <--- 26 520-539 185 185 185 334 334 334 T T F <--- 27 540-559 184 184 184 333 333 333 F F F 28 560-579 184 184 184 333 333 333 T T F <--- 29 580-599 184 184 184 333 333 333 T T F <--- Max gen: 188, Max flush: 335 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 325 325 325 F F F 1 020-039 188 188 188 335 335 335 F F F 2 040-059 177 177 177 318 318 318 F F F 3 060-079 189 189 189 336 336 336 F F F 4 080-099 189 189 189 336 336 336 F F F 5 100-119 189 189 189 336 336 336 F F F 6 120-139 185 185 185 334 334 334 F F F 7 140-159 185 185 185 334 334 334 F F F 8 160-179 189 189 189 336 336 336 F F F 9 180-199 189 189 189 336 336 336 F F F 10 200-219 185 185 185 334 334 334 F F F 11 220-239 185 185 185 334 334 334 F F F 12 240-259 189 189 189 336 336 336 F F F 13 260-279 189 189 189 336 336 336 F F F 14 280-299 189 189 189 336 336 336 F F F 15 300-319 189 189 189 336 336 336 F F F 16 320-339 185 185 185 334 334 334 F F F 17 340-359 185 185 185 334 334 334 F F F 18 360-379 185 185 185 334 334 334 F F F 19 380-399 189 189 189 336 336 336 F F F 20 400-419 185 185 185 334 334 334 F F F 21 420-439 178 178 178 320 320 320 F F F 22 440-459 182 182 182 329 329 329 F F F 23 460-479 185 185 185 334 334 334 F F F 24 480-499 185 185 185 334 334 334 F F F 25 500-519 189 189 189 336 336 336 F F F 26 520-539 189 189 189 336 336 336 F F F 27 540-559 184 184 184 333 333 333 F F F 28 560-579 189 189 189 336 336 336 F F F 29 580-599 189 189 189 336 336 336 F F F Max gen: 189, Max flush: 336 Begin loop 95 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.533062475Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.560822786Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.561400109Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.561484697Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.561520478Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.561567882Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955} {"msg":"Crucible 6f808394-acaa-4843-ac86-bcfeefb38e5c has session id: 5064c87a-4f2f-4e92-9aad-7cf960b40d35","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.5626682Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"Upstairs opts: Upstairs UUID: 6f808394-acaa-4843-ac86-bcfeefb38e5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.562727467Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"Crucible stats registered with UUID: 6f808394-acaa-4843-ac86-bcfeefb38e5c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.562753279Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:34.562781745Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.563237948Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.563371352Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.563416381Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.564028734Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.564407213Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955} {"msg":"6f808394-acaa-4843-ac86-bcfeefb38e5c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.56445094Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.564674122Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"client":"0","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.564726676Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"client":"1","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.564768609Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"client":"2","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.564808758Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.564848206Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.565001199Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"ds_connection connected from Ok(127.0.0.1:51395)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.565154873Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"io task","client":"2","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"ds_connection connected from Ok(127.0.0.1:59293)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.565473073Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"io task","client":"0","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"ds_connection connected from Ok(127.0.0.1:41375)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.565535517Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"io task","client":"1","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.567082951Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"client":"0","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.567132539Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"client":"1","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.568359608Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.568434376Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"client":"2","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.568465167Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"[0]R flush_numbers[0..12]: [325, 335, 318, 336, 336, 336, 334, 334, 336, 336, 334, 334]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569278718Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"downstairs","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"[0]R generation[0..12]: [180, 188, 177, 189, 189, 189, 185, 185, 189, 189, 185, 185]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569327565Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"downstairs","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.56936055Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"downstairs","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"[1]R flush_numbers[0..12]: [325, 335, 318, 336, 336, 336, 334, 334, 336, 336, 334, 334]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569391672Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"downstairs","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"[1]R generation[0..12]: [180, 188, 177, 189, 189, 189, 185, 185, 189, 189, 185, 185]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569422232Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"downstairs","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569454085Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"downstairs","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"[2]R flush_numbers[0..12]: [325, 335, 318, 336, 336, 336, 334, 334, 336, 336, 334, 334]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569484896Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"downstairs","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"[2]R generation[0..12]: [180, 188, 177, 189, 189, 189, 185, 185, 189, 189, 185, 185]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569515256Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"downstairs","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.56954746Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"downstairs","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569580325Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"downstairs","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569607699Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"downstairs","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569633611Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"downstairs","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569659462Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.56969912Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.56972431Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"client":"0","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569751103Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"client":"1","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569777946Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"client":"2","":"downstairs","session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569805371Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569830961Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"6f808394-acaa-4843-ac86-bcfeefb38e5c is now active with session: 5064c87a-4f2f-4e92-9aad-7cf960b40d35","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569859919Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569884097Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955,"session_id":"5064c87a-4f2f-4e92-9aad-7cf960b40d35"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569909617Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955} {"msg":"Activated sub_volume 6f808394-acaa-4843-ac86-bcfeefb38e5c","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:34.569931571Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 233 len 16384 data: 19 21 20 23 02/30 Write block 68 len 24576 data: 14 16 15 13 14 14 03/30 Write block 218 len 28672 data: 12 11 13 11 9 7 8 04/30 Read block 284 len 12288 05/30 Write block 514 len 28672 data: 16 14 13 12 13 14 11 06/30 Read block 363 len 8192 07/30 Read block 428 len 16384 08/30 Flush 09/30 Write block 270 len 20480 data: 17 17 17 17 19 10/30 Flush 11/30 Write block 368 len 40960 data: 14 12 13 13 12 12 13 9 11 12 12/30 Read block 590 len 4096 13/30 Read block 245 len 36864 14/30 Read block 378 len 28672 15/30 Read block 126 len 32768 16/30 Write block 187 len 8192 data: 15 16 17/30 Write block 475 len 32768 data: 15 13 13 12 12 14 13 16 18/30 Write block 104 len 8192 data: 9 10 19/30 Write block 286 len 24576 data: 13 13 16 17 17 22 20/30 Write block 309 len 16384 data: 21 19 17 15 21/30 Read block 141 len 24576 22/30 Read block 2 len 4096 23/30 Write block 575 len 12288 data: 17 15 17 24/30 Read block 343 len 16384 25/30 Read block 111 len 24576 26/30 Write block 46 len 8192 data: 13 11 27/30 Write block 332 len 16384 data: 9 10 13 10 28/30 Write block 3 len 16384 data: 14 17 19 19 29/30 Read block 149 len 16384 30/30 Write block 245 len 24576 data: 11 11 10 10 12 10 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 10 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Read 6 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 15 1 17 Done 19 5 19 43 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:34.600763444Z","hostname":"bmat-EVT22200007-0000ead9","pid":1955} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 325 325 325 T F T <--- 1 020-039 188 188 188 335 335 335 F F F 2 040-059 177 177 177 318 318 318 T F T <--- 3 060-079 190 190 190 337 337 337 F F F 4 080-099 189 189 189 336 336 336 F F F 5 100-119 189 189 189 336 336 336 T F T <--- 6 120-139 185 185 185 334 334 334 F F F 7 140-159 185 185 185 334 334 334 F F F 8 160-179 189 189 189 336 336 336 F F F 9 180-199 189 189 189 336 336 336 T F T <--- 10 200-219 190 190 190 337 337 337 F F F 11 220-239 190 190 190 337 337 337 F F F 12 240-259 189 189 189 336 336 336 T F T <--- 13 260-279 190 190 190 338 338 338 F F F 14 280-299 189 189 189 336 336 336 T F T <--- 15 300-319 189 189 189 336 336 336 T F T <--- 16 320-339 185 185 185 334 334 334 T F T <--- 17 340-359 185 185 185 334 334 334 F F F 18 360-379 185 185 185 334 334 334 T T T 19 380-399 189 189 189 336 336 336 F F F 20 400-419 185 185 185 334 334 334 F F F 21 420-439 178 178 178 320 320 320 F F F 22 440-459 182 182 182 329 329 329 F F F 23 460-479 185 185 185 334 334 334 T F T <--- 24 480-499 185 185 185 334 334 334 T F T <--- 25 500-519 190 190 190 337 337 337 F F F 26 520-539 190 190 190 337 337 337 F F F 27 540-559 184 184 184 333 333 333 F F F 28 560-579 189 189 189 336 336 336 T F T <--- 29 580-599 189 189 189 336 336 336 F F F Max gen: 190, Max flush: 338 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 339 339 339 F F F 1 020-039 188 188 188 335 335 335 F F F 2 040-059 191 191 191 339 339 339 F F F 3 060-079 190 190 190 337 337 337 F F F 4 080-099 189 189 189 336 336 336 F F F 5 100-119 191 191 191 339 339 339 F F F 6 120-139 185 185 185 334 334 334 F F F 7 140-159 185 185 185 334 334 334 F F F 8 160-179 189 189 189 336 336 336 F F F 9 180-199 191 191 191 339 339 339 F F F 10 200-219 190 190 190 337 337 337 F F F 11 220-239 190 190 190 337 337 337 F F F 12 240-259 191 191 191 339 339 339 F F F 13 260-279 190 190 190 338 338 338 F F F 14 280-299 191 191 191 339 339 339 F F F 15 300-319 191 191 191 339 339 339 F F F 16 320-339 191 191 191 339 339 339 F F F 17 340-359 185 185 185 334 334 334 F F F 18 360-379 191 191 191 339 339 339 F F F 19 380-399 189 189 189 336 336 336 F F F 20 400-419 185 185 185 334 334 334 F F F 21 420-439 178 178 178 320 320 320 F F F 22 440-459 182 182 182 329 329 329 F F F 23 460-479 191 191 191 339 339 339 F F F 24 480-499 191 191 191 339 339 339 F F F 25 500-519 190 190 190 337 337 337 F F F 26 520-539 190 190 190 337 337 337 F F F 27 540-559 184 184 184 333 333 333 F F F 28 560-579 191 191 191 339 339 339 F F F 29 580-599 189 189 189 336 336 336 F F F Max gen: 191, Max flush: 339 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.288758533Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.315331313Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.316093352Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.316233169Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.316280152Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.316327496Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963} {"msg":"Crucible b3313afa-f1b5-445c-b836-f45f287810d2 has session id: 8312e596-66bf-4595-98a8-e314ab2e9f36","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.317492182Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"Upstairs opts: Upstairs UUID: b3313afa-f1b5-445c-b836-f45f287810d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.317609825Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"Crucible stats registered with UUID: b3313afa-f1b5-445c-b836-f45f287810d2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.317642259Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:39.31766793Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.31789546Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.317972573Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.31806215Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.318871593Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.319190735Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963} {"msg":"b3313afa-f1b5-445c-b836-f45f287810d2 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.319243148Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.319408395Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"client":"0","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.319439958Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"client":"1","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.319467412Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"client":"2","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.319509496Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.319537992Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"ds_connection connected from Ok(127.0.0.1:45750)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.3197389Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"io task","client":"0","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.319792646Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"ds_connection connected from Ok(127.0.0.1:62776)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.319988524Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"io task","client":"2","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"ds_connection connected from Ok(127.0.0.1:62625)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.320336954Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"io task","client":"1","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.321734571Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"client":"0","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.321904648Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"client":"2","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.322610395Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.322721886Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"client":"1","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.322774189Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"[0]R flush_numbers[0..12]: [339, 335, 339, 337, 336, 339, 334, 334, 336, 339, 337, 337]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.323801613Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"downstairs","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"[0]R generation[0..12]: [191, 188, 191, 190, 189, 191, 185, 185, 189, 191, 190, 190]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.323872394Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"downstairs","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.323917433Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"downstairs","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"[1]R flush_numbers[0..12]: [339, 335, 339, 337, 336, 339, 334, 334, 336, 339, 337, 337]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.323957863Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"downstairs","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"[1]R generation[0..12]: [191, 188, 191, 190, 189, 191, 185, 185, 189, 191, 190, 190]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.3240069Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"downstairs","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324049685Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"downstairs","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"[2]R flush_numbers[0..12]: [339, 335, 339, 337, 336, 339, 334, 334, 336, 339, 337, 337]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324089063Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"downstairs","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"[2]R generation[0..12]: [191, 188, 191, 190, 189, 191, 185, 185, 189, 191, 190, 190]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324127088Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"downstairs","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324168039Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"downstairs","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324202156Z","hostname":"bmat-EVT22200007-0000ead9","pidWait for a query_work_queue command to finish before sending IO ":1963,"":"downstairs","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324251614Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"downstairs","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"Next flush: 340","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324286073Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"downstairs","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324319619Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"":"downstairs","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324354478Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324386752Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"client":"0","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324421831Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"client":"1","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.32446166Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"client":"2","":"downstairs","session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324503082Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324543532Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"b3313afa-f1b5-445c-b836-f45f287810d2 is now active with session: 8312e596-66bf-4595-98a8-e314ab2e9f36","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.32457782Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324604914Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963,"session_id":"8312e596-66bf-4595-98a8-e314ab2e9f36"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324633611Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963} {"msg":"Activated sub_volume b3313afa-f1b5-445c-b836-f45f287810d2","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:39.324658109Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 97 len 28672 data: 23 20 19 19 11 12 7 02/30 Read block 560 len 12288 03/30 Read block 142 len 8192 04/30 Read block 434 len 20480 05/30 Read block 303 len 24576 06/30 Write block 212 len 20480 data: 15 11 10 9 12 07/30 Read block 168 len 40960 08/30 Write block 566 len 12288 data: 16 18 15 09/30 Write block 299 len 28672 data: 10 12 11 15 17 18 17 10/30 Write block 450 len 20480 data: 13 11 10 9 13 11/30 Read block 347 len 28672 12/30 Write block 311 len 20480 data: 18 16 15 13 16 13/30 Read block 174 len 8192 14/30 Read block 433 len 32768 15/30 Read block 65 len 36864 16/30 Write block 458 len 24576 data: 19 19 18 14 17 18 17/30 Flush 18/30 Write block 531 len 20480 data: 18 16 18 16 17 19/30 Read block 249 len 40960 20/30 Read block 295 len 40960 21/30 Read block 309 len 24576 22/30 Flush 23/30 Write block 42 len 20480 data: 11 15 15 18 14 24/30 Flush 25/30 Write block 453 len 32768 data: 10 14 11 13 16 20 20 19 26/30 Flush 27/30 Write block 573 len 12288 data: 12 14 18 28/30 Write block 234 len 12288 data: 22 21 24 29/30 Write block 317 len 36864 data: 14 14 15 20 17 18 17 15 10 30/30 Write block 179 len 16384 data: 16 19 14 14 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Write 5 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 19 2 23 Done 28 11 28 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:39.358586443Z","hostname":"bmat-EVT22200007-0000ead9","pid":1963} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 339 339 339 F F F 1 020-039 188 188 188 335 335 335 F F F 2 040-059 192 191 192 342 339 342 F F F <--- 3 060-079 190 190 190 337 337 337 F F F 4 080-099 192 189 192 340 336 340 F T F <--- 5 100-119 192 191 192 340 339 340 F T F <--- 6 120-139 185 185 185 334 334 334 F F F 7 140-159 185 185 185 334 334 334 F F F 8 160-179 189 189 189 336 336 336 T F T <--- 9 180-199 191 191 191 339 339 339 T F T <--- 10 200-219 192 190 192 340 337 340 F T F <--- 11 220-239 190 190 190 337 337 337 T F T <--- 12 240-259 191 191 191 339 339 339 F F F 13 260-279 190 190 190 338 338 338 F F F 14 280-299 192 191 192 340 339 340 F T F <--- 15 300-319 192 191 192 340 339 340 T T T <--- 16 320-339 191 191 191 339 339 339 T F T <--- 17 340-359 185 185 185 334 334 334 F F F 18 360-379 191 191 191 339 339 339 F F F 19 380-399 189 189 189 336 336 336 F F F 20 400-419 185 185 185 334 334 334 F F F 21 420-439 178 178 178 320 320 320 F F F 22 440-459 192 182 192 343 329 343 F T F <--- 23 460-479 192 191 192 343 339 343 F F F <--- 24 480-499 191 191 191 339 339 339 F F F 25 500-519 190 190 190 337 337 337 F F F 26 520-539 192 190 192 341 337 341 F F F <--- 27 540-559 184 184 184 333 333 333 F F F 28 560-579 192 191 192 340 339 340 T T T <--- 29 580-599 189 189 189 336 336 336 F F F Max gen: 192, Max flush: 343 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 339 339 339 F F F 1 020-039 188 188 188 335 335 335 F F F 2 040-059 192 192 192 342 342 342 F F F 3 060-079 190 190 190 337 337 337 F F F 4 080-099 192 192 192 340 340 340 F F F 5 100-119 192 192 192 340 340 340 F F F 6 120-139 185 185 185 334 334 334 F F F 7 140-159 185 185 185 334 334 334 F F F 8 160-179 193 193 193 344 344 344 F F F 9 180-199 193 193 193 344 344 344 F F F 10 200-219 192 192 192 340 340 340 F F F 11 220-239 193 193 193 344 344 344 F F F 12 240-259 191 191 191 339 339 339 F F F 13 260-279 190 190 190 338 338 338 F F F 14 280-299 192 192 192 340 340 340 F F F 15 300-319 193 193 193 344 344 344 F F F 16 320-339 193 193 193 344 344 344 F F F 17 340-359 185 185 185 334 334 334 F F F 18 360-379 191 191 191 339 339 339 F F F 19 380-399 189 189 189 336 336 336 F F F 20 400-419 185 185 185 334 334 334 F F F 21 420-439 178 178 178 320 320 320 F F F 22 440-459 192 192 192 343 343 343 F F F 23 460-479 192 192 192 343 343 343 F F F 24 480-499 191 191 191 339 339 339 F F F 25 500-519 190 190 190 337 337 337 F F F 26 520-539 192 192 192 341 341 341 F F F 27 540-559 184 184 184 333 333 333 F F F 28 560-579 193 193 193 344 344 344 F F F 29 580-599 189 189 189 336 336 336 F F F Max gen: 193, Max flush: 344 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.026016446Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.050638164Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.051160147Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.051240607Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.051265165Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.051303752Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971} {"msg":"Crucible 20d989cd-08e2-4876-b29b-6ca4bc9735ce has session id: 1fd76279-9574-4e16-9369-92632e4e79b6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.052215928Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"Upstairs opts: Upstairs UUID: 20d989cd-08e2-4876-b29b-6ca4bc9735ce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.052278972Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"Crucible stats registered with UUID: 20d989cd-08e2-4876-b29b-6ca4bc9735ce","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.052310445Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:44.052337849Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.052745276Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.052800244Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"2","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.052831717Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"0","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.052970041Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.053000301Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971} {"msg":"20d989cd-08e2-4876-b29b-6ca4bc9735ce active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.053109928Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.053140558Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"client":"0","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.053169977Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"client":"1","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.053198984Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"client":"2","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.053228332Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"2","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.053258422Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"0","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.053356105Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:36990)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.053583976Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"0","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:60561)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.053625649Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"2","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:44.053664977Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:44.053702962Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:44.053747891Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.053966904Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"sleeping for 10s before connecting","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.054010972Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.054932155Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"client":"2","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.055078956Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"client":"0","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.055805104Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:44.055902847Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.054930436Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:65186)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.055064031Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"io task","client":"1","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.057894335Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"client":"1","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"[0]R flush_numbers[0..12]: [339, 335, 342, 337, 340, 340, 334, 334, 344, 344, 340, 344]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.058787292Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"downstairs","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"[0]R generation[0..12]: [191, 188, 192, 190, 192, 192, 185, 185, 193, 193, 192, 193]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.058844927Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"downstairs","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.058882431Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"downstairs","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"[1]R flush_numbers[0..12]: [339, 335, 342, 337, 340, 340, 334, 334, 344, 344, 340, 344]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.058917079Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"downstairs","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"[1]R generation[0..12]: [191, 188, 192, 190, 192, 192, 185, 185, 193, 193, 192, 193]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.058969102Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"downstairs","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059004813Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"downstairs","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"[2]R flush_numbers[0..12]: [339, 335, 342, 337, 340, 340, 334, 334, 344, 344, 340, 344]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059038891Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"downstairs","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"[2]R generation[0..12]: [191, 188, 192, 190, 192, 192, 185, 185, 193, 193, 192, 193]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059073489Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"downstairs","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059108689Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"downstairs","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059137586Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"downstairs","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059167575Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"downstairs","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059195781Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"downstairs","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059225139Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"":"downstairs","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059254798Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059281591Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"client":"0","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.05931143Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"client":"1","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059341069Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"client":"2","":"downstairs","session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059371519Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059399615Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"20d989cd-08e2-4876-b29b-6ca4bc9735ce is now active with session: 1fd76279-9574-4e16-9369-92632e4e79b6","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059431418Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059461387Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971,"session_id":"1fd76279-9574-4e16-9369-92632e4e79b6"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-04T01:17:54.059489623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971} {"msg":"Activated sub_volume 20d989cd-08e2-4876-b29b-6ca4bc9735ce","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:54.059524151Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 41 len 36864 data: 11 12 16 16 19 15 12 9 12 02/30 Read block 354 len 12288 03/30 Read block 138 len 36864 04/30 Read block 330 len 12288 05/30 Write block 321 len 36864 data: 18 19 18 16 11 12 10 10 10 06/30 Read block 33 len 16384 07/30 Read block 317 len 36864 08/30 Read block 321 len 8192 09/30 Write block 460 len 16384 data: 20 15 18 19 10/30 Read block 213 len 36864 11/30 Write block 34 len 12288 data: 12 11 14 12/30 Read block 236 len 16384 13/30 Write block 168 len 32768 data: 15 12 11 12 11 11 14 14 14/30 Read block 272 len 32768 15/30 Write block 529 len 12288 data: 21 19 19 16/30 Flush 17/30 Write block 223 len 4096 data: 8 18/30 Read block 293 len 8192 19/30 Write block 315 len 40960 data: 17 17 15 15 16 21 19 20 19 17 20/30 Read block 322 len 16384 21/30 Write block 110 len 40960 data: 14 15 12 19 17 17 20 22 20 17 22/30 Write block 429 len 40960 data: 14 14 14 10 13 12 17 20 23 22 23/30 Write block 376 len 40960 data: 12 13 13 14 13 12 12 11 12 14 24/30 Write block 332 len 36864 data: 10 11 14 11 13 15 13 13 14 25/30 Write block 169 len 28672 data: 13 12 13 12 12 15 15 26/30 Flush 27/30 Write block 116 len 40960 data: 21 23 21 18 18 15 13 11 12 12 28/30 Write block 494 len 20480 data: 15 14 16 16 15 29/30 Write block 408 len 32768 data: 17 13 14 12 10 10 11 12 30/30 Read block 503 len 12288 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Write 4 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 22 0 22 Done 30 8 30 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:54.101594325Z","hostname":"bmat-EVT22200007-0000ead9","pid":1971} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 339 339 339 F F F 1 020-039 194 188 194 345 335 345 F F F <--- 2 040-059 194 192 194 345 342 345 F T F <--- 3 060-079 190 190 190 337 337 337 F F F 4 080-099 192 192 192 340 340 340 F F F 5 100-119 194 192 194 346 340 346 T F T <--- 6 120-139 185 185 185 334 334 334 T F T <--- 7 140-159 185 185 185 334 334 334 F F F 8 160-179 194 193 194 346 344 346 F F F <--- 9 180-199 193 193 193 344 344 344 F F F 10 200-219 192 192 192 340 340 340 F F F 11 220-239 194 193 194 346 344 346 F F F <--- 12 240-259 191 191 191 339 339 339 F F F 13 260-279 190 190 190 338 338 338 F F F 14 280-299 192 192 192 340 340 340 F F F 15 300-319 194 193 194 346 344 346 F F F <--- 16 320-339 194 193 194 346 344 346 F T F <--- 17 340-359 194 185 194 346 334 346 F F F <--- 18 360-379 194 191 194 346 339 346 F F F <--- 19 380-399 194 189 194 346 336 346 F F F <--- 20 400-419 185 185 185 334 334 334 T F T <--- 21 420-439 194 178 194 346 320 346 F F F <--- 22 440-459 192 192 192 343 343 343 F F F 23 460-479 194 192 194 345 343 345 F F F <--- 24 480-499 191 191 191 339 339 339 T F T <--- 25 500-519 190 190 190 337 337 337 F F F 26 520-539 194 192 194 345 341 345 F F F <--- 27 540-559 184 184 184 333 333 333 F F F 28 560-579 193 193 193 344 344 344 F F F 29 580-599 189 189 189 336 336 336 F F F Max gen: 194, Max flush: 346 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 339 339 339 F F F 1 020-039 194 194 194 345 345 345 F F F 2 040-059 194 194 194 345 345 345 F F F 3 060-079 190 190 190 337 337 337 F F F 4 080-099 192 192 192 340 340 340 F F F 5 100-119 195 195 195 347 347 347 F F F 6 120-139 195 195 195 347 347 347 F F F 7 140-159 185 185 185 334 334 334 F F F 8 160-179 194 194 194 346 346 346 F F F 9 180-199 193 193 193 344 344 344 F F F 10 200-219 192 192 192 340 340 340 F F F 11 220-239 194 194 194 346 346 346 F F F 12 240-259 191 191 191 339 339 339 F F F 13 260-279 190 190 190 338 338 338 F F F 14 280-299 192 192 192 340 340 340 F F F 15 300-319 194 194 194 346 346 346 F F F 16 320-339 194 194 194 346 346 346 F F F 17 340-359 194 194 194 346 346 346 F F F 18 360-379 194 194 194 346 346 346 F F F 19 380-399 194 194 194 346 346 346 F F F 20 400-419 195 195 195 347 347 347 F F F 21 420-439 194 194 194 346 346 346 F F F 22 440-459 192 192 192 343 343 343 F F F 23 460-479 194 194 194 345 345 345 F F F 24 480-499 195 195 195 347 347 347 F F F 25 500-519 190 190 190 337 337 337 F F F 26 520-539 194 194 194 345 345 345 F F F 27 540-559 184 184 184 333 333 333 F F F 28 560-579 193 193 193 344 344 344 F F F 29 580-599 189 189 189 336 336 336 F F F Max gen: 195, Max flush: 347 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.769765356Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:58.788529796Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.788614815Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.807526197Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.808176304Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.80830092Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.808328936Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.808366099Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979} {"msg":"Crucible 6ac989f4-c026-47f8-a368-06055e6c849b has session id: d4cb9a94-5de2-4d27-b0f0-2796af86d370","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.809450987Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"Upstairs opts: Upstairs UUID: 6ac989f4-c026-47f8-a368-06055e6c849b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.809528451Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"Crucible stats registered with UUID: 6ac989f4-c026-47f8-a368-06055e6c849b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.809567217Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:58.809593209Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.810056346Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.810129861Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.810158749Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.81042203Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.810464023Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979} {"msg":"6ac989f4-c026-47f8-a368-06055e6c849b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.810568941Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.810602056Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"client":"0","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.810629781Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"client":"1","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.810656534Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"client":"2","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.810683447Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.810858123Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"ds_connection connected from Ok(127.0.0.1:59739)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.810935737Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"io task","client":"2","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.810968572Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"ds_connection connected from Ok(127.0.0.1:32962)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.811316721Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"io task","client":"1","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"ds_connection connected from Ok(127.0.0.1:36924)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.811367081Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"io task","client":"0","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.812595022Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"client":"2","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.812747895Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"client":"1","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.813527168Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.813781501Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.814136704Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"client":"0","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"[0]R flush_numbers[0..12]: [339, 345, 345, 337, 340, 347, 347, 334, 346, 344, 340, 346]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.814985906Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"downstairs","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"[0]R generation[0..12]: [191, 194, 194, 190, 192, 195, 195, 185, 194, 193, 192, 194]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815025083Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"downstairs","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815057698Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"downstairs","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"[1]R flush_numbers[0..12]: [339, 345, 345, 337, 340, 347, 347, 334, 346, 344, 340, 346]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815120101Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"downstairs","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"[1]R generation[0..12]: [191, 194, 194, 190, 192, 195, 195, 185, 194, 193, 192, 194]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815161634Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"downstairs","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815193587Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"downstairs","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"[2]R flush_numbers[0..12]: [339, 345, 345, 337, 340, 347, 347, 334, 346, 344, 340, 346]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815224478Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"downstairs","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"[2]R generation[0..12]: [191, 194, 194, 190, 192, 195, 195, 185, 194, 193, 192, 194]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815255239Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"downstairs","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815287513Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"downstairs","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.81532125Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"downstairs","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815348924Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"downstairs","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"Next flush: 348","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815374645Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"downstairs","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815400226Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"":"downstairs","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.81542732Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815451858Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"client":"0","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815478862Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"client":"1","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815505545Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"client":"2","":"downstairs","session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815534291Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815559632Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"6ac989f4-c026-47f8-a368-06055e6c849b is now active with session: d4cb9a94-5de2-4d27-b0f0-2796af86d370","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815588328Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815612266Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979,"session_id":"d4cb9a94-5de2-4d27-b0f0-2796af86d370"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.815637877Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979} {"msg":"Activated sub_volume 6ac989f4-c026-47f8-a368-06055e6c849b","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:17:58.81565983Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 146 len 20480 data: 14 16 16 11 11 02/30 Write block 463 len 12288 data: 20 16 15 03/30 Write block 376 len 40960 data: 13 14 14 15 14 13 13 12 13 15 04/30 Read block 161 len 40960 05/30 Write block 220 len 32768 data: 14 12 10 9 9 8 9 11 06/30 Write block 106 len 32768 data: 8 8 12 11 15 16 13 20 07/30 Write block 242 len 36864 data: 14 13 11 12 12 11 11 13 11 08/30 Read block 314 len 36864 09/30 Read block 129 len 20480 10/30 Flush 11/30 Write block 433 len 4096 data: 14 12/30 Read block 116 len 24576 13/30 Read block 103 len 8192 14/30 Read block 123 len 8192 15/30 Flush 16/30 Write block 252 len 40960 data: 9 11 13 13 12 16 16 15 18 19 17/30 Read block 552 len 4096 18/30 Write block 547 len 36864 data: 12 11 8 11 14 17 17 18 18 19/30 Write block 566 len 12288 data: 17 19 16 20/30 Flush 21/30 Write block 515 len 16384 data: 15 14 13 14 22/30 Read block 487 len 16384 23/30 Write block 461 len 40960 data: 16 19 21 17 16 16 16 13 14 15 24/30 Read block 359 len 40960 25/30 Write block 174 len 8192 data: 16 16 26/30 Read block 470 len 36864 27/30 Read block 573 len 40960 28/30 Read block 504 len 12288 29/30 Write block 190 len 36864 data: 14 17 17 17 17 15 17 18 19 30/30 Read block 489 len 4096 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 3 Sent Done Done false 1002 Acked Write 10 Sent Done Done false 1003 Acked Read 10 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Write 10 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:17:58.855132621Z","hostname":"bmat-EVT22200007-0000ead9","pid":1979} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 339 339 339 F F F 1 020-039 194 194 194 345 345 345 F F F 2 040-059 194 194 194 345 345 345 F F F 3 060-079 190 190 190 337 337 337 F F F 4 080-099 192 192 192 340 340 340 F F F 5 100-119 195 196 196 347 348 348 F F F <--- 6 120-139 195 195 195 347 347 347 F F F 7 140-159 185 196 196 334 348 348 T F F <--- 8 160-179 194 194 194 346 346 346 F T T <--- 9 180-199 193 193 193 344 344 344 F T T <--- 10 200-219 192 192 192 340 340 340 F F F 11 220-239 194 196 196 346 348 348 F F F <--- 12 240-259 191 196 196 339 350 350 F F F <--- 13 260-279 190 196 196 338 350 350 F F F <--- 14 280-299 192 192 192 340 340 340 F F F 15 300-319 194 194 194 346 346 346 F F F 16 320-339 194 194 194 346 346 346 F F F 17 340-359 194 194 194 346 346 346 F F F 18 360-379 194 196 196 346 348 348 F F F <--- 19 380-399 194 196 196 346 348 348 F F F <--- 20 400-419 195 195 195 347 347 347 F F F 21 420-439 194 196 196 346 349 349 F F F <--- 22 440-459 192 192 192 343 343 343 F F F 23 460-479 194 196 196 345 348 348 F T T <--- 24 480-499 195 195 195 347 347 347 F F F 25 500-519 190 190 190 337 337 337 F T T <--- 26 520-539 194 194 194 345 345 345 F F F 27 540-559 184 196 196 333 350 350 F F F <--- 28 560-579 193 196 196 344 350 350 F F F <--- 29 580-599 189 189 189 336 336 336 F F F Max gen: 196, Max flush: 350 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 191 191 339 339 339 F F F 1 020-039 194 194 194 345 345 345 F F F 2 040-059 194 194 194 345 345 345 F F F 3 060-079 190 190 190 337 337 337 F F F 4 080-099 192 192 192 340 340 340 F F F 5 100-119 196 196 196 348 348 348 F F F 6 120-139 195 195 195 347 347 347 F F F 7 140-159 196 196 196 348 348 348 F F F 8 160-179 197 197 197 351 351 351 F F F 9 180-199 197 197 197 351 351 351 F F F 10 200-219 192 192 192 340 340 340 F F F 11 220-239 196 196 196 348 348 348 F F F 12 240-259 196 196 196 350 350 350 F F F 13 260-279 196 196 196 350 350 350 F F F 14 280-299 192 192 192 340 340 340 F F F 15 300-319 194 194 194 346 346 346 F F F 16 320-339 194 194 194 346 346 346 F F F 17 340-359 194 194 194 346 346 346 F F F 18 360-379 196 196 196 348 348 348 F F F 19 380-399 196 196 196 348 348 348 F F F 20 400-419 195 195 195 347 347 347 F F F 21 420-439 196 196 196 349 349 349 F F F 22 440-459 192 192 192 343 343 343 F F F 23 460-479 197 197 197 351 351 351 F F F 24 480-499 195 195 195 347 347 347 F F F 25 500-519 197 197 197 351 351 351 F F F 26 520-539 194 194 194 345 345 345 F F F 27 540-559 196 196 196 350 350 350 F F F 28 560-579 196 196 196 350 350 350 F F F 29 580-599 189 189 189 336 336 336 F F F Max gen: 197, Max flush: 351 Begin loop 99 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.524358Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:18:03.550243279Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.550348557Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.575129821Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.575875537Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"6b9c36a99f72e4eb3754717a90ae2049ec668b64\",\n git_commit_timestamp: \"2026-03-04T00:43:48.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.575960265Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.575986277Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.576025574Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987} {"msg":"Crucible c6c04a71-dd15-4630-9e2d-c3f1ba8afc61 has session id: 51a7d372-45cd-48ca-82b4-27ddb1cfdcd3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.577479974Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"Upstairs opts: Upstairs UUID: c6c04a71-dd15-4630-9e2d-c3f1ba8afc61, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.577551305Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"Crucible stats registered with UUID: c6c04a71-dd15-4630-9e2d-c3f1ba8afc61","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.577583699Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:18:03.577611695Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.57821527Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"io task","client":"0","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.578511987Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"io task","client":"2","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.578602025Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"io task","client":"1","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.578645692Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.578682294Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987} {"msg":"c6c04a71-dd15-4630-9e2d-c3f1ba8afc61 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.578878623Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.578921418Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"client":"0","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.578957579Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"client":"1","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.579047117Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"client":"2","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.579232514Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"io task","client":"2","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.579271611Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"io task","client":"0","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.579305819Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"io task","client":"1","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"ds_connection connected from Ok(127.0.0.1:53032)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.579346409Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"io task","client":"1","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45778)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.579400586Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"io task","client":"2","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"ds_connection connected from Ok(127.0.0.1:38404)","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.579438742Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"io task","client":"0","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.580936678Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"client":"2","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.581053238Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"client":"1","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.581689719Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.581839756Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.582316019Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"client":"0","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"[0]R flush_numbers[0..12]: [339, 345, 345, 337, 340, 348, 347, 348, 351, 351, 340, 348]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583177905Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"downstairs","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"[0]R generation[0..12]: [191, 194, 194, 190, 192, 196, 195, 196, 197, 197, 192, 196]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583229247Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"downstairs","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583270468Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"downstairs","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"[1]R flush_numbers[0..12]: [339, 345, 345, 337, 340, 348, 347, 348, 351, 351, 340, 348]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583309035Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"downstairs","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"[1]R generation[0..12]: [191, 194, 194, 190, 192, 196, 195, 196, 197, 197, 192, 196]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583347711Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"downstairs","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583388362Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"downstairs","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"[2]R flush_numbers[0..12]: [339, 345, 345, 337, 340, 348, 347, 348, 351, 351, 340, 348]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583427569Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"downstairs","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"[2]R generation[0..12]: [191, 194, 194, 190, 192, 196, 195, 196, 197, 197, 192, 196]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583466516Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"downstairs","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583506325Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"downstairs","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583539541Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"downstairs","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.58357462Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"downstairs","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"Next flush: 352","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583607255Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"downstairs","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583639779Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"":"downstairs","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583674398Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.5837055Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"client":"0","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583745589Z","hostname":"Wait for a query_work_queue command to finish before sending IO bmat-EVT22200007-0000ead9","pid":1987,"client":"1","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.58379679Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"client":"2","":"downstairs","session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.58383179Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583864354Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"c6c04a71-dd15-4630-9e2d-c3f1ba8afc61 is now active with session: 51a7d372-45cd-48ca-82b4-27ddb1cfdcd3","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583901027Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583931888Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987,"session_id":"51a7d372-45cd-48ca-82b4-27ddb1cfdcd3"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583963761Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987} {"msg":"Activated sub_volume c6c04a71-dd15-4630-9e2d-c3f1ba8afc61","v":0,"name":"crucible","level":30,"time":"2026-03-04T01:18:03.583991776Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 402 len 16384 data: 15 16 18 17 02/30 Write block 179 len 32768 data: 17 20 15 15 13 14 15 15 03/30 Read block 480 len 4096 04/30 Write block 212 len 8192 data: 16 12 05/30 Write block 69 len 32768 data: 17 16 14 15 15 12 13 10 06/30 Flush 07/30 Write block 149 len 24576 data: 12 12 14 12 14 15 08/30 Read block 85 len 28672 09/30 Write block 15 len 8192 data: 12 9 10/30 Write block 89 len 36864 data: 16 13 11 13 16 15 17 21 24 11/30 Read block 71 len 32768 12/30 Read block 44 len 36864 13/30 Read block 437 len 4096 14/30 Write block 438 len 28672 data: 23 18 17 13 14 11 13 15/30 Read block 332 len 16384 16/30 Read block 539 len 16384 17/30 Flush 18/30 Write block 394 len 32768 data: 13 11 13 13 10 12 16 17 19/30 Read block 442 len 24576 20/30 Read block 475 len 24576 21/30 Read block 57 len 28672 22/30 Write block 527 len 40960 data: 16 19 22 20 20 17 19 17 18 19 23/30 Write block 427 len 12288 data: 12 11 15 24/30 Read block 180 len 4096 25/30 Write block 98 len 32768 data: 21 20 20 12 13 8 10 11 26/30 Flush 27/30 Write block 355 len 40960 data: 7 10 17 22 21 18 19 21 18 18 28/30 Read block 519 len 28672 29/30 Write block 522 len 36864 data: 17 18 18 17 16 17 20 23 21 30/30 Read block 16 len 24576 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 8 Sent Done Done false 1002 Acked Read 1 Sent Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 8 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-04T01:18:03.621935339Z","hostname":"bmat-EVT22200007-0000ead9","pid":1987} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 191 198 198 339 353 353 F F F <--- 1 020-039 194 194 194 345 345 345 F F F 2 040-059 194 194 194 345 345 345 F F F 3 060-079 190 198 198 337 352 352 F F F <--- 4 080-099 192 198 198 340 354 354 F F F <--- 5 100-119 196 198 198 348 354 354 F F F <--- 6 120-139 195 195 195 347 347 347 F F F 7 140-159 196 198 198 348 353 353 F F F <--- 8 160-179 197 198 198 351 352 352 F F F <--- 9 180-199 197 198 198 351 352 352 F F F <--- 10 200-219 192 198 198 340 352 352 F F F <--- 11 220-239 196 196 196 348 348 348 F F F 12 240-259 196 196 196 350 350 350 F F F 13 260-279 196 196 196 350 350 350 F F F 14 280-299 192 192 192 340 340 340 F F F 15 300-319 194 194 194 346 346 346 F F F 16 320-339 194 194 194 346 346 346 F F F 17 340-359 194 194 194 346 346 346 F T T <--- 18 360-379 196 196 196 348 348 348 F T T <--- 19 380-399 196 198 198 348 354 354 F F F <--- 20 400-419 195 198 198 347 354 354 T F F <--- 21 420-439 196 198 198 349 354 354 F F F <--- 22 440-459 192 198 198 343 353 353 F F F <--- 23 460-479 197 197 197 351 351 351 F F F 24 480-499 195 195 195 347 347 347 F F F 25 500-519 197 197 197 351 351 351 F F F 26 520-539 194 198 198 345 354 354 F T T <--- 27 540-559 196 196 196 350 350 350 F F F 28 560-579 196 196 196 350 350 350 F F F 29 580-599 189 189 189 336 336 336 F F F Max gen: 198, Max flush: 354 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/backtrace.rs:27:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100/src/lib.rs:695:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 353 353 353 F F F 1 020-039 194 194 194 345 345 345 F F F 2 040-059 194 194 194 345 345 345 F F F 3 060-079 198 198 198 352 352 352 F F F 4 080-099 198 198 198 354 354 354 F F F 5 100-119 198 198 198 354 354 354 F F F 6 120-139 195 195 195 347 347 347 F F F 7 140-159 198 198 198 353 353 353 F F F 8 160-179 198 198 198 352 352 352 F F F 9 180-199 198 198 198 352 352 352 F F F 10 200-219 198 198 198 352 352 352 F F F 11 220-239 196 196 196 348 348 348 F F F 12 240-259 196 196 196 350 350 350 F F F 13 260-279 196 196 196 350 350 350 F F F 14 280-299 192 192 192 340 340 340 F F F 15 300-319 194 194 194 346 346 346 F F F 16 320-339 194 194 194 346 346 346 F F F 17 340-359 199 199 199 355 355 355 F F F 18 360-379 199 199 199 355 355 355 F F F 19 380-399 198 198 198 354 354 354 F F F 20 400-419 198 198 198 354 354 354 F F F 21 420-439 198 198 198 354 354 354 F F F 22 440-459 198 198 198 353 353 353 F F F 23 460-479 197 197 197 351 351 351 F F F 24 480-499 195 195 195 347 347 347 F F F 25 500-519 197 197 197 351 351 351 F F F 26 520-539 199 199 199 355 355 355 F F F 27 540-559 196 196 196 350 350 350 F F F 28 560-579 196 196 196 350 350 350 F F F 29 580-599 189 189 189 336 336 336 F F F Max gen: 199, Max flush: 355 11:05 Test duration Test completed real 11:04.973819532 user 1:18.676683641 sys 40.415627699 trap 0.173196168 tflt 0.250552597 dflt 0.321394250 kflt 0.015391497 lock 17:20:17.665660669 slp 52:58.739607451 lat 15.696486927 stop 0.062308168