Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.19859403 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.013947152 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.012373136 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1134 1135 1136 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.487418343Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.503475848Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.506005723Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.506133318Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.506142777Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.506154645Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139} {"msg":"Crucible 16e69298-9819-4389-958c-c763dfabf4bc has session id: 294456e1-c1a5-4239-acd1-cf7bb1159324","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.507076858Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"Upstairs opts: Upstairs UUID: 16e69298-9819-4389-958c-c763dfabf4bc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.507360825Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"Crucible stats registered with UUID: 16e69298-9819-4389-958c-c763dfabf4bc","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.507371853Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:15.507420938Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.508105188Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.508369247Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.508382775Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.508718876Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.5087738Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139} {"msg":"16e69298-9819-4389-958c-c763dfabf4bc active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.508783009Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.508792018Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"client":"0","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.508801846Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"client":"1","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.508815095Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"client":"2","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.508824854Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.508834773Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.508866709Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"ds_connection connected from Ok(127.0.0.1:54171)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.508899515Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"ds_connection connected from Ok(127.0.0.1:33897)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.508915463Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"ds_connection connected from Ok(127.0.0.1:60843)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.509153155Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.510833459Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"client":"1","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.510861426Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"client":"2","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.510875244Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"client":"0","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.511843161Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.511892616Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512008012Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"downstairs","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512019151Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"downstairs","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.51203009Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"downstairs","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512040508Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"downstairs","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512050667Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"downstairs","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512061676Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"downstairs","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512071805Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"downstairs","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512081834Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"downstairs","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512092762Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"downstairs","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512102281Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"downstairs","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.51211207Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"downstairs","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512121699Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"downstairs","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512132548Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"downstairs","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512142537Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512151475Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"client":"0","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512161334Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"client":"1","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512171043Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"client":"2","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512181092Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512190251Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"16e69298-9819-4389-958c-c763dfabf4bc is now active with session: 294456e1-c1a5-4239-acd1-cf7bb1159324","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.51220021Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512209019Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512218088Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139} {"msg":"Activated sub_volume 16e69298-9819-4389-958c-c763dfabf4bc","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.51271218Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.512748076Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.513100975Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"task":"crutest"} Read and Verify all blocks (0..600 range:false) {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30---------------------------------------------------------------- ,"time":"2026-09-16T04:39:15.553985762Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"task":"crutest"} {"msg":"CLIENT: Tests done. All submitted work has been ACK'd","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.554022118Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"task":"crutest"} Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 256 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 88 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 {"msg":"CLIENT: Up:0 ds:3 act:3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:15.554162262Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"task":"crutest"} ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 {"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.55489689Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"task":"crutest"} {"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.554935855Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"task":"crutest"} {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:19.554945364Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139} {"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:19.554997058Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:19.555054472Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:19.555081098Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.555092017Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"client stopping due to Err(RecvError(()))","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.555103256Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:19.555183956Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:19.555209923Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.555220982Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"294456e1-c1a5-4239-acd1-cf7bb1159324"} Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.578107871Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.586293615Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.586568763Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.58659359Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.586602289Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.586613838Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141} {"msg":"Crucible 6ff8b9ed-2f1c-4c51-99be-ce700dc13166 has session id: c18f3018-dace-4b44-bf4c-79140bdcc41e","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.586949198Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"Upstairs opts: Upstairs UUID: 6ff8b9ed-2f1c-4c51-99be-ce700dc13166, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.586973596Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"Crucible stats registered with UUID: 6ff8b9ed-2f1c-4c51-99be-ce700dc13166","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.586983275Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:19.587007812Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587153525Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587168693Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587178662Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587323185Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587345962Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141} {"msg":"6ff8b9ed-2f1c-4c51-99be-ce700dc13166 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587386307Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587396286Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"client":"0","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587408475Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"client":"1","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587418194Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"client":"2","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587427943Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587462239Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587484436Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"ds_connection connected from Ok(127.0.0.1:32871)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587628869Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"ds_connection connected from Ok(127.0.0.1:53027)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587654386Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38379)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.587665775Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588210111Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"client":"0","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588235678Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"client":"1","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588361444Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"client":"2","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.58856412Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588610635Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588635542Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"downstairs","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.58864613Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"downstairs","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588659079Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"downstairs","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588669208Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"downstairs","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588679337Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"downstairs","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588690305Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"downstairs","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588700544Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"downstairs","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588710473Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"downstairs","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588721232Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"downstairs","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588730581Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"downstairs","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588740079Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"downstairs","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588749448Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"downstairs","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588758717Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"":"downstairs","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588768456Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588777335Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"client":"0","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588787014Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"client":"1","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588796853Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"client":"2","":"downstairs","session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588807992Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.58881703Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"6ff8b9ed-2f1c-4c51-99be-ce700dc13166 is now active with session: c18f3018-dace-4b44-bf4c-79140bdcc41e","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588826919Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588835578Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"session_id":"c18f3018-dace-4b44-bf4c-79140bdcc41e"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588844517Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141} {"msg":"Activated sub_volume 6ff8b9ed-2f1c-4c51-99be-ce700dc13166","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588852516Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.588860475Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.589375915Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"task":"crutest"} 01/30 Write block 71 len 24576 data: 2 2 2 2 2 2 02/30 Read block 1 len 28672 03/30 Read block 347 len 40960 04/30 Write block 57 len 4096 data: 2 05/30 Read block 170 len 8192 06/30 Read block 535 len 16384 07/30 Read block 77 len 16384 08/30 Read block 398 len 8192 09/30 Write block 78 len 8192 data: 2 2 10/30 Write block 208 len 20480 data: 2 2 2 2 2 11/30 Read block 198 len 24576 12/30 Read block 153 len 8192 13/30 Flush 14/30 Write block 504 len 20480 data: 2 2 2 2 2 15/30 Read block 470 len 24576 16/30 Read block 265 len 12288 17/30 Write block 280 len 32768 data: 2 2 2 2 2 2 2 2 18/30 Write block 572 len 16384 data: 2 2 2 2 19/30 Write block 459 len 40960 data: 2 2 2 2 2 2 2 2 2 2 20/30 Read block 278 len 28672 21/30 Write block 78 len 12288 data: 3 3 2 22/30 Flush 23/30 Write block 44 len 16384 data: 2 2 2 2 24/30 Read block 344 len 28672 25/30 Write block 95 len 4096 data: 2 26/30 Read block 565 len 28672 27/30 Write block 118 len 32768 data: 2 2 2 2 2 2 2 2 28/30 Write block 441 len 12288 data: 2 2 2 29/30 Write block 45 len 28672 data: 3 3 3 2 2 2 2 30/30 Read block 0 len 16384 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Write 2 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 22 23 Done 30 29 8 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:19.598036834Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:19.598210074Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1141,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 1 2 2 1 T T T <--- 3 060-079 2 2 1 3 3 1 F F T <--- 4 080-099 2 2 1 3 3 1 T T F <--- 5 100-119 1 1 1 1 1 1 T T F <--- 6 120-139 1 1 1 1 1 1 T T F <--- 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 2 2 1 2 2 1 F F F <--- 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 2 2 1 3 3 1 F F F <--- 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 2 2 1 3 3 1 T T F <--- 23 460-479 2 2 1 3 3 1 F F F <--- 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 1 3 3 1 F F F <--- 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 1 3 3 1 F F F <--- 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 3 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 3 3 3 4 4 4 F F F 3 060-079 2 2 2 3 3 3 F F F 4 080-099 3 3 3 4 4 4 F F F 5 100-119 3 3 3 4 4 4 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 2 2 2 2 2 2 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 2 2 2 3 3 3 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 2 2 2 3 3 3 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 3 3 3 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 4 Begin loop 2 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.977222486Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.98532829Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.985608187Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.985637334Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.985645773Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.985657191Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149} {"msg":"Crucible 87a87bb7-2488-4ec4-a37d-6f20c6d7f4ff has session id: 23bfd240-d096-4e47-8e5d-1c7d775ba8a7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.985970295Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"Upstairs opts: Upstairs UUID: 87a87bb7-2488-4ec4-a37d-6f20c6d7f4ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986002581Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"Crucible stats registered with UUID: 87a87bb7-2488-4ec4-a37d-6f20c6d7f4ff","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986015809Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:23.986025078Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986139965Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986154953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986164942Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986316224Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986331353Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149} {"msg":"87a87bb7-2488-4ec4-a37d-6f20c6d7f4ff active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.98635494Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986364509Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"client":"0","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986374098Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"client":"1","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986383597Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"client":"2","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986393155Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986403114Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986413083Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:40457)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986501113Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:63799)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.98652793Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:63316)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.986539288Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987241176Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"client":"1","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987284661Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"client":"0","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987307209Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"client":"2","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987579337Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987589606Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 4, 3, 4, 4, 4, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987608964Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"downstairs","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"[0]R generation[0..12]: [1, 1, 3, 2, 3, 3, 3, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987619302Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"downstairs","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987630211Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"downstairs","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 4, 3, 4, 4, 4, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.98764041Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"downstairs","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"[1]R generation[0..12]: [1, 1, 3, 2, 3, 3, 3, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987650969Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"downstairs","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987662047Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"downstairs","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 4, 3, 4, 4, 4, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987672216Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"downstairs","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"[2]R generation[0..12]: [1, 1, 3, 2, 3, 3, 3, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987682345Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"downstairs","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987694534Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"downstairs","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987703962Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"downstairs","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987713581Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"downstairs","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.98772277Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"downstairs","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987732179Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"":"downstairs","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987741928Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987750737Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"client":"0","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987760466Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"client":"1","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987770035Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"client":"2","":"downstairs","session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987779864Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987789063Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"87a87bb7-2488-4ec4-a37d-6f20c6d7f4ff is now active with session: 23bfd240-d096-4e47-8e5d-1c7d775ba8a7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987798981Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.98780789Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"session_id":"23bfd240-d096-4e47-8e5d-1c7d775ba8a7"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987816949Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149} {"msg":"Activated sub_volume 87a87bb7-2488-4ec4-a37d-6f20c6d7f4ff","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987824858Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.987833007Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.988142091Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"task":"crutest"} 01/30 Flush 02/30 Write block 589 len 8192 data: 2 2 03/30 Write block 521 len 4096 data: 2 04/30 Write block 51 len 32768 data: 3 2 2 2 2 2 3 2 05/30 Flush 06/30 Write block 579 len 40960 data: 2 2 2 2 2 2 2 2 2 2 07/30 Write block 455 len 24576 data: 2 2 2 2 3 3 08/30 Write block 77 len 28672 data: 2 4 4 3 2 2 2 09/30 Write block 12 len 32768 data: 2 2 2 2 2 2 2 2 10/30 Read block 193 len 36864 11/30 Flush 12/30 Write block 515 len 28672 data: 2 2 2 2 2 2 3 13/30 Write block 476 len 28672 data: 2 2 2 2 2 2 2 14/30 Write block 64 len 40960 data: 2 2 2 2 2 2 2 3 3 3 15/30 Flush 16/30 Write block 3 len 8192 data: 2 2 17/30 Read block 527 len 40960 18/30 Read block 150 len 8192 19/30 Write block 300 len 40960 data: 2 2 2 2 2 2 2 2 2 2 20/30 Write block 492 len 20480 data: 2 2 2 2 2 21/30 Read block 375 len 24576 22/30 Read block 212 len 20480 23/30 Read block 55 len 16384 24/30 Read block 36 len 24576 25/30 Flush 26/30 Write block 126 len 12288 data: 2 2 2 27/30 Read block 123 len 20480 28/30 Read block 253 len 28672 29/30 Read block 378 len 20480 30/30 Write block 550 len 40960 data: 2 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 2 Done Done Sent false 1002 Acked Write 1 Done Done Sent false 1003 Acked Write 8 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 10 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Write 8 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:23.996455561Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:23.996669866Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1149,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 1 9 9 1 F F F <--- 1 020-039 1 1 1 1 1 1 F F F 2 040-059 4 4 3 6 6 4 F F F <--- 3 060-079 4 4 2 8 8 3 F F F <--- 4 080-099 4 4 3 7 7 4 F F F <--- 5 100-119 3 3 3 4 4 4 F F F 6 120-139 3 3 3 4 4 4 T T F <--- 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 2 2 2 2 2 2 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 4 4 1 9 9 1 F F F <--- 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 4 4 3 7 7 4 F F F <--- 23 460-479 4 4 2 8 8 3 F F F <--- 24 480-499 4 4 1 9 9 1 F F F <--- 25 500-519 4 4 2 8 8 3 F F F <--- 26 520-539 4 4 1 8 8 1 F F F <--- 27 540-559 1 1 1 1 1 1 T T F <--- 28 560-579 4 4 2 7 7 3 F F F <--- 29 580-599 4 4 1 7 7 1 F F F <--- Max gen: 4, Max flush: 9 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 9 9 9 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 4 4 4 6 6 6 F F F 3 060-079 4 4 4 8 8 8 F F F 4 080-099 4 4 4 7 7 7 F F F 5 100-119 3 3 3 4 4 4 F F F 6 120-139 5 5 5 10 10 10 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 2 2 2 2 2 2 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 4 4 4 9 9 9 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 4 4 4 8 8 8 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 4 4 4 8 8 8 F F F 27 540-559 5 5 5 10 10 10 F F F 28 560-579 4 4 4 7 7 7 F F F 29 580-599 4 4 4 7 7 7 F F F Max gen: 5, Max flush: 10 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.236843055Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.244888766Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245180082Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245214398Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245223707Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245235355Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157} {"msg":"Crucible 96381c58-87b0-460e-940d-07b87824a2ed has session id: 618bf7f2-e3c8-4aae-9855-021c51b6e315","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245511493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"Upstairs opts: Upstairs UUID: 96381c58-87b0-460e-940d-07b87824a2ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245528851Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"Crucible stats registered with UUID: 96381c58-87b0-460e-940d-07b87824a2ed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.24553878Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:28.245548029Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245686623Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245701661Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.24571152Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245855103Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.24588187Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157} {"msg":"96381c58-87b0-460e-940d-07b87824a2ed active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245893168Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245901967Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"client":"0","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245911726Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"client":"1","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245921315Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"client":"2","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245930904Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245941053Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245950952Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"ds_connection connected from Ok(127.0.0.1:40707)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.245989087Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"ds_connection connected from Ok(127.0.0.1:53620)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.246020274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"ds_connection connected from Ok(127.0.0.1:60422)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.246031072Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.246644261Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"client":"0","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.246662309Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"client":"1","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.246673067Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"client":"2","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.246932657Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.246965903Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"[0]R flush_numbers[0..12]: [9, 1, 6, 8, 7, 4, 10, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247013058Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"downstairs","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"[0]R generation[0..12]: [4, 1, 4, 4, 4, 3, 5, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247023936Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"downstairs","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247034865Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"downstairs","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"[1]R flush_numbers[0..12]: [9, 1, 6, 8, 7, 4, 10, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247045204Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"downstairs","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"[1]R generation[0..12]: [4, 1, 4, 4, 4, 3, 5, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247055293Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"downstairs","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247066131Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"downstairs","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"[2]R flush_numbers[0..12]: [9, 1, 6, 8, 7, 4, 10, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.24707632Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"downstairs","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"[2]R generation[0..12]: [4, 1, 4, 4, 4, 3, 5, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247086449Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"downstairs","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247097348Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"downstairs","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247106737Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"downstairs","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247116386Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"downstairs","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247125705Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"downstairs","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247135263Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"":"downstairs","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247145362Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247155771Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"client":"0","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.24716572Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"client":"1","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247175479Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"client":"2","":"downstairs","session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247185308Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247194447Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"96381c58-87b0-460e-940d-07b87824a2ed is now active with session: 618bf7f2-e3c8-4aae-9855-021c51b6e315","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247204325Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247213014Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"session_id":"618bf7f2-e3c8-4aae-9855-021c51b6e315"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247222143Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157} {"msg":"Activated sub_volume 96381c58-87b0-460e-940d-07b87824a2ed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247230092Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247238031Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.247521368Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"task":"crutest"} 01/30 Write block 150 len 24576 data: 2 2 2 2 2 2 02/30 Read block 500 len 28672 03/30 Read block 549 len 24576 04/30 Write block 553 len 12288 data: 3 3 3 05/30 Write block 273 len 40960 data: 2 2 2 2 2 2 2 3 3 3 06/30 Write block 535 len 32768 data: 2 2 2 2 2 2 2 2 07/30 Read block 132 len 40960 08/30 Read block 186 len 36864 09/30 Write block 492 len 16384 data: 3 3 3 3 10/30 Write block 117 len 4096 data: 2 11/30 Write block 555 len 40960 data: 4 3 3 3 3 2 2 2 2 2 12/30 Write block 293 len 16384 data: 2 2 2 2 13/30 Read block 15 len 32768 14/30 Read block 461 len 4096 15/30 Read block 496 len 8192 16/30 Read block 146 len 8192 17/30 Read block 151 len 12288 18/30 Read block 391 len 12288 19/30 Flush 20/30 Write block 53 len 40960 data: 3 3 3 3 4 3 2 2 2 2 21/30 Write block 153 len 20480 data: 3 3 3 2 2 22/30 Write block 35 len 8192 data: 2 2 23/30 Write block 405 len 28672 data: 2 2 2 2 2 2 2 24/30 Write block 198 len 20480 data: 2 2 2 2 2 25/30 Write block 549 len 8192 data: 2 3 26/30 Write block 529 len 32768 data: 2 2 2 2 2 2 3 3 27/30 Write block 126 len 4096 data: 3 28/30 Read block 471 len 40960 29/30 Write block 554 len 12288 data: 4 5 4 30/30 Write block 188 len 24576 data: 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Write 10 Done Done Done false 1005 Acked Write 8 Done Done Sent false 1006 Acked Read 10 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Write 4 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Read 1 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 25 29 Done 28 28 5 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:28.254662595Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:28.254868791Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1157,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 9 9 9 F F F 1 020-039 1 1 1 1 1 1 T T F <--- 2 040-059 4 4 4 6 6 6 T T F <--- 3 060-079 4 4 4 8 8 8 T T F <--- 4 080-099 4 4 4 7 7 7 F F F 5 100-119 6 6 3 11 11 4 F F F <--- 6 120-139 5 5 5 10 10 10 T T F <--- 7 140-159 6 6 1 11 11 1 T T T <--- 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 T T F <--- 10 200-219 2 2 2 2 2 2 T T F <--- 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 6 6 1 11 11 1 F F T <--- 14 280-299 6 6 2 11 11 3 F F T <--- 15 300-319 4 4 4 9 9 9 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 T T F <--- 21 420-439 1 1 1 1 1 1 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 4 4 4 8 8 8 F F F 24 480-499 6 6 4 11 11 9 F F F <--- 25 500-519 4 4 4 8 8 8 F F F 26 520-539 6 6 4 11 11 8 T T F <--- 27 540-559 6 6 5 11 11 10 T T T <--- 28 560-579 6 6 4 11 11 7 F F F <--- 29 580-599 4 4 4 7 7 7 F F F Max gen: 6, Max flush: 11 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 9 9 9 F F F 1 020-039 7 7 7 12 12 12 F F F 2 040-059 7 7 7 12 12 12 F F F 3 060-079 7 7 7 12 12 12 F F F 4 080-099 4 4 4 7 7 7 F F F 5 100-119 6 6 6 11 11 11 F F F 6 120-139 7 7 7 12 12 12 F F F 7 140-159 7 7 7 12 12 12 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 7 7 7 12 12 12 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 6 6 6 11 11 11 F F F 14 280-299 6 6 6 11 11 11 F F F 15 300-319 4 4 4 9 9 9 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 7 7 7 12 12 12 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 4 4 4 7 7 7 F F F 23 460-479 4 4 4 8 8 8 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 7 7 7 12 12 12 F F F 27 540-559 7 7 7 12 12 12 F F F 28 560-579 6 6 6 11 11 11 F F F 29 580-599 4 4 4 7 7 7 F F F Max gen: 7, Max flush: 12 Begin loop 4 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.508308211Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:32.515566094Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.515616348Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.522530171Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.522790201Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.522824557Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.522836675Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.522848444Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165} {"msg":"Crucible fc29d6fb-6005-4bc2-99ed-97ca8c278632 has session id: 17f9a264-146a-463e-a71b-699fb2315b57","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523178665Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"Upstairs opts: Upstairs UUID: fc29d6fb-6005-4bc2-99ed-97ca8c278632, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523243208Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"Crucible stats registered with UUID: fc29d6fb-6005-4bc2-99ed-97ca8c278632","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523268055Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:32.523277944Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523403729Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523431956Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523442614Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523574769Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523586678Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165} {"msg":"fc29d6fb-6005-4bc2-99ed-97ca8c278632 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523603586Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523612495Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"client":"0","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523622054Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"client":"1","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523633252Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"client":"2","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523642821Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.52365275Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523662879Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"ds_connection connected from Ok(127.0.0.1:48818)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523745839Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"ds_connection connected from Ok(127.0.0.1:35929)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523757128Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"ds_connection connected from Ok(127.0.0.1:64956)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.523767307Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524411471Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"client":"2","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524437738Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"client":"1","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524466485Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"client":"0","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524699568Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524727844Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"[0]R flush_numbers[0..12]: [9, 12, 12, 12, 7, 11, 12, 12, 1, 12, 12, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524814454Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"downstairs","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"[0]R generation[0..12]: [4, 7, 7, 7, 4, 6, 7, 7, 1, 7, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524825863Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"downstairs","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524836832Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"downstairs","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"[1]R flush_numbers[0..12]: [9, 12, 12, 12, 7, 11, 12, 12, 1, 12, 12, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.52485001Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"downstairs","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"[1]R generation[0..12]: [4, 7, 7, 7, 4, 6, 7, 7, 1, 7, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524860479Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"downstairs","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524871638Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"downstairs","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"[2]R flush_numbers[0..12]: [9, 12, 12, 12, 7, 11, 12, 12, 1, 12, 12, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524881916Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"downstairs","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"[2]R generation[0..12]: [4, 7, 7, 7, 4, 6, 7, 7, 1, 7, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524891955Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"downstairs","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524902674Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"downstairs","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524912093Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"downstairs","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524921782Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"downstairs","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524930941Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"downstairs","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.52494037Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"":"downstairs","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524950139Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524958937Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"client":"0","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524968636Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"client":"1","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524978205Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"client":"2","":"downstairs","session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.524987914Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.525014231Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"fc29d6fb-6005-4bc2-99ed-97ca8c278632 is now active with session: 17f9a264-146a-463e-a71b-699fb2315b57","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.52502432Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.525032959Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"session_id":"17f9a264-146a-463e-a71b-699fb2315b57"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.525041918Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165} {"msg":"Activated sub_volume fc29d6fb-6005-4bc2-99ed-97ca8c278632","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.525049897Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.525057926Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.525326585Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"task":"crutest"} 01/30 Write block 543 len 20480 data: 2 2 2 2 2 02/30 Read block 149 len 36864 03/30 Read block 359 len 4096 04/30 Write block 354 len 4096 data: 2 05/30 Read block 281 len 16384 06/30 Flush 07/30 Write block 146 len 20480 data: 2 2 2 2 3 08/30 Write block 333 len 4096 data: 2 09/30 Read block 444 len 36864 10/30 Read block 333 len 24576 11/30 Read block 344 len 28672 12/30 Flush 13/30 Write block 575 len 4096 data: 3 14/30 Read block 383 len 36864 15/30 Flush 16/30 Write block 416 len 4096 data: 2 17/30 Write block 354 len 4096 data: 3 18/30 Read block 417 len 28672 19/30 Read block 415 len 16384 20/30 Write block 448 len 16384 data: 2 2 2 2 21/30 Read block 260 len 20480 22/30 Write block 368 len 20480 data: 2 2 2 2 2 23/30 Write block 426 len 32768 data: 2 2 2 2 2 2 2 2 24/30 Read block 128 len 36864 25/30 Write block 589 len 40960 data: 3 3 2 2 2 2 2 2 2 2 26/30 Read block 186 len 24576 27/30 Read block 464 len 28672 28/30 Write block 532 len 12288 data: 3 3 3 29/30 Write block 81 len 28672 data: 3 3 3 2 2 2 2 30/30 Write block 80 len 20480 data: 4 4 4 4 3 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Write 1 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 1 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Write 1 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 2 2 31 Done 3 28 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:32.533705247Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:32.533858649Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1165,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 9 9 9 F F F 1 020-039 7 7 7 12 12 12 F F F 2 040-059 7 7 7 12 12 12 F F F 3 060-079 7 7 7 12 12 12 F F F 4 080-099 4 4 4 7 7 7 F T T <--- 5 100-119 6 6 6 11 11 11 F F F 6 120-139 7 7 7 12 12 12 F F F 7 140-159 7 8 8 12 14 14 F F F <--- 8 160-179 1 1 1 1 1 1 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 7 7 7 12 12 12 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 6 6 6 11 11 11 F F F 14 280-299 6 6 6 11 11 11 F F F 15 300-319 4 4 4 9 9 9 F F F 16 320-339 1 8 8 1 14 14 F F F <--- 17 340-359 1 8 8 1 13 13 F T T <--- 18 360-379 1 1 1 1 1 1 F T T <--- 19 380-399 1 1 1 1 1 1 F F F 20 400-419 7 7 7 12 12 12 F T T <--- 21 420-439 1 1 1 1 1 1 F T T <--- 22 440-459 4 4 4 7 7 7 F T T <--- 23 460-479 4 4 4 8 8 8 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 7 7 7 12 12 12 F T T <--- 27 540-559 7 8 8 12 13 13 T F F <--- 28 560-579 6 8 8 11 15 15 F F F <--- 29 580-599 4 4 4 7 7 7 F T T <--- Max gen: 8, Max flush: 15 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 9 9 9 F F F 1 020-039 7 7 7 12 12 12 F F F 2 040-059 7 7 7 12 12 12 F F F 3 060-079 7 7 7 12 12 12 F F F 4 080-099 9 9 9 16 16 16 F F F 5 100-119 6 6 6 11 11 11 F F F 6 120-139 7 7 7 12 12 12 F F F 7 140-159 8 8 8 14 14 14 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 7 7 7 12 12 12 F F F 10 200-219 7 7 7 12 12 12 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 6 6 6 11 11 11 F F F 14 280-299 6 6 6 11 11 11 F F F 15 300-319 4 4 4 9 9 9 F F F 16 320-339 8 8 8 14 14 14 F F F 17 340-359 9 9 9 16 16 16 F F F 18 360-379 9 9 9 16 16 16 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 9 9 9 16 16 16 F F F 21 420-439 9 9 9 16 16 16 F F F 22 440-459 9 9 9 16 16 16 F F F 23 460-479 4 4 4 8 8 8 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 9 9 9 16 16 16 F F F 27 540-559 8 8 8 13 13 13 F F F 28 560-579 8 8 8 15 15 15 F F F 29 580-599 9 9 9 16 16 16 F F F Max gen: 9, Max flush: 16 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.781674655Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.790221588Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.790523143Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.79054953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.790558288Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.790569787Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173} {"msg":"Crucible 61dfa9ce-8587-4594-9af3-8b3236c44b16 has session id: b1dc3fa0-496a-427d-8b60-3f45332adf64","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.790934495Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"Upstairs opts: Upstairs UUID: 61dfa9ce-8587-4594-9af3-8b3236c44b16, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.790963381Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"Crucible stats registered with UUID: 61dfa9ce-8587-4594-9af3-8b3236c44b16","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.79097445Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:36.790986039Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.791112724Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.79114133Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.791152249Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.791260697Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.791279974Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173} {"msg":"61dfa9ce-8587-4594-9af3-8b3236c44b16 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.791306201Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.79131569Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"client":"0","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.791325409Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"client":"1","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.791335168Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"client":"2","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.791344777Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.791354686Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.791364714Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"ds_connection connected from Ok(127.0.0.1:57546)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.791383372Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"ds_connection connected from Ok(127.0.0.1:40026)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.791409679Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"ds_connection connected from Ok(127.0.0.1:54828)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.791532755Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792063783Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"client":"0","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792096239Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"client":"1","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792199127Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"client":"2","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792376816Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792392754Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"[0]R flush_numbers[0..12]: [9, 12, 12, 12, 16, 11, 12, 14, 1, 12, 12, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792487903Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"downstairs","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"[0]R generation[0..12]: [4, 7, 7, 7, 9, 6, 7, 8, 1, 7, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792499052Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"downstairs","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792509991Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"downstairs","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"[1]R flush_numbers[0..12]: [9, 12, 12, 12, 16, 11, 12, 14, 1, 12, 12, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792520499Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"downstairs","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"[1]R generation[0..12]: [4, 7, 7, 7, 9, 6, 7, 8, 1, 7, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792530568Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"downstairs","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792541537Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"downstairs","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"[2]R flush_numbers[0..12]: [9, 12, 12, 12, 16, 11, 12, 14, 1, 12, 12, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792551776Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"downstairs","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"[2]R generation[0..12]: [4, 7, 7, 7, 9, 6, 7, 8, 1, 7, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792561865Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"downstairs","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792572813Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"downstairs","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792584932Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"downstairs","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792594741Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"downstairs","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.79260416Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"downstairs","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792613709Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"":"downstairs","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792623547Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792632376Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"client":"0","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792642065Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"client":"1","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792651654Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"client":"2","":"downstairs","session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792661633Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792670782Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"61dfa9ce-8587-4594-9af3-8b3236c44b16 is now active with session: b1dc3fa0-496a-427d-8b60-3f45332adf64","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792680671Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.79268945Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"session_id":"b1dc3fa0-496a-427d-8b60-3f45332adf64"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792698529Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173} {"msg":"Activated sub_volume 61dfa9ce-8587-4594-9af3-8b3236c44b16","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792706438Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.792714527Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.793007463Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"task":"crutest"} 01/30 Write block 87 len 40960 data: 3 2 2 2 2 2 2 2 3 2 02/30 Read block 378 len 32768 03/30 Flush 04/30 Write block 426 len 36864 data: 3 3 3 3 3 3 3 3 2 05/30 Write block 102 len 28672 data: 2 2 2 2 2 2 2 06/30 Flush 07/30 Write block 158 len 8192 data: 2 2 08/30 Read block 465 len 12288 09/30 Flush 10/30 Write block 382 len 40960 data: 2 2 2 2 2 2 2 2 2 2 11/30 Write block 387 len 24576 data: 3 3 3 3 3 2 12/30 Write block 337 len 40960 data: 2 2 2 2 2 2 2 2 2 2 13/30 Write block 198 len 4096 data: 3 14/30 Write block 570 len 28672 data: 2 2 3 3 3 4 2 15/30 Read block 77 len 32768 16/30 Read block 82 len 28672 17/30 Read block 539 len 24576 18/30 Read block 227 len 20480 19/30 Write block 360 len 40960 data: 2 2 2 2 2 2 2 2 3 3 20/30 Read block 109 len 24576 21/30 Read block 474 len 32768 22/30 Read block 173 len 40960 23/30 Flush 24/30 Flush 25/30 Write block 0 len 4096 data: 2 26/30 Read block 424 len 24576 27/30 Read block 508 len 36864 28/30 Write block 533 len 28672 data: 4 4 4 4 3 3 3 29/30 Read block 275 len 32768 30/30 Read block 131 len 24576 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 8 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Write 7 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 6 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:36.802325345Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:36.802499955Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1173,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 9 9 9 T T F <--- 1 020-039 7 7 7 12 12 12 F F F 2 040-059 7 7 7 12 12 12 F F F 3 060-079 7 7 7 12 12 12 F F F 4 080-099 10 10 9 17 17 16 F F T <--- 5 100-119 10 10 6 18 18 11 F F F <--- 6 120-139 7 7 7 12 12 12 F F F 7 140-159 10 10 8 19 19 14 F F F <--- 8 160-179 1 1 1 1 1 1 F F F 9 180-199 10 10 7 20 20 12 F F F <--- 10 200-219 7 7 7 12 12 12 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 6 6 6 11 11 11 F F F 14 280-299 6 6 6 11 11 11 F F F 15 300-319 4 4 4 9 9 9 F F F 16 320-339 10 10 8 20 20 14 F F F <--- 17 340-359 10 10 9 20 20 16 F F F <--- 18 360-379 10 10 9 20 20 16 F F F <--- 19 380-399 10 10 1 20 20 1 F F F <--- 20 400-419 9 9 9 16 16 16 F F F 21 420-439 10 10 9 18 18 16 F F F <--- 22 440-459 9 9 9 16 16 16 F F F 23 460-479 4 4 4 8 8 8 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 9 9 9 16 16 16 T T F <--- 27 540-559 8 8 8 13 13 13 F F F 28 560-579 10 10 8 20 20 15 F F F <--- 29 580-599 9 9 9 16 16 16 F F F Max gen: 10, Max flush: 20 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 21 21 21 F F F 1 020-039 7 7 7 12 12 12 F F F 2 040-059 7 7 7 12 12 12 F F F 3 060-079 7 7 7 12 12 12 F F F 4 080-099 10 10 10 17 17 17 F F F 5 100-119 10 10 10 18 18 18 F F F 6 120-139 7 7 7 12 12 12 F F F 7 140-159 10 10 10 19 19 19 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 10 10 10 20 20 20 F F F 10 200-219 7 7 7 12 12 12 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 6 6 6 11 11 11 F F F 14 280-299 6 6 6 11 11 11 F F F 15 300-319 4 4 4 9 9 9 F F F 16 320-339 10 10 10 20 20 20 F F F 17 340-359 10 10 10 20 20 20 F F F 18 360-379 10 10 10 20 20 20 F F F 19 380-399 10 10 10 20 20 20 F F F 20 400-419 9 9 9 16 16 16 F F F 21 420-439 10 10 10 18 18 18 F F F 22 440-459 9 9 9 16 16 16 F F F 23 460-479 4 4 4 8 8 8 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 8 8 8 13 13 13 F F F 28 560-579 10 10 10 20 20 20 F F F 29 580-599 9 9 9 16 16 16 F F F Max gen: 11, Max flush: 21 Begin loop 6 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.041407041Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.049439313Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.0497202Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.049746147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.049754796Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.049766365Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181} {"msg":"Crucible 50cad5d6-fc20-465f-a69c-ff265d243452 has session id: dd74d965-c748-4986-af3e-8e065ffd7c67","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050134542Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"Upstairs opts: Upstairs UUID: 50cad5d6-fc20-465f-a69c-ff265d243452, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050163559Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"Crucible stats registered with UUID: 50cad5d6-fc20-465f-a69c-ff265d243452","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050174467Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:41.050183746Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050339428Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050368995Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050379843Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050544994Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050560312Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181} {"msg":"50cad5d6-fc20-465f-a69c-ff265d243452 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050593948Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050605487Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"client":"0","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050615256Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"client":"1","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050624825Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"client":"2","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050634314Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050644003Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050653801Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"ds_connection connected from Ok(127.0.0.1:59465)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050741251Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"ds_connection connected from Ok(127.0.0.1:47392)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050757989Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"ds_connection connected from Ok(127.0.0.1:59660)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.050902372Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051424391Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"client":"0","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051446779Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"client":"1","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051457868Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"client":"2","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051728866Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051738855Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"[0]R flush_numbers[0..12]: [21, 12, 12, 12, 17, 18, 12, 19, 1, 20, 12, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051771841Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"downstairs","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"[0]R generation[0..12]: [11, 7, 7, 7, 10, 10, 7, 10, 1, 10, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051784829Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"downstairs","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051795958Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"downstairs","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"[1]R flush_numbers[0..12]: [21, 12, 12, 12, 17, 18, 12, 19, 1, 20, 12, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051806527Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"downstairs","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"[1]R generation[0..12]: [11, 7, 7, 7, 10, 10, 7, 10, 1, 10, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051816856Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"downstairs","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051828014Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"downstairs","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"[2]R flush_numbers[0..12]: [21, 12, 12, 12, 17, 18, 12, 19, 1, 20, 12, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051838643Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"downstairs","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"[2]R generation[0..12]: [11, 7, 7, 7, 10, 10, 7, 10, 1, 10, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051848712Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"downstairs","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051859651Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"downstairs","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.05186919Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"downstairs","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051879068Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"downstairs","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051888507Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"downstairs","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051897926Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"":"downstairs","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051907645Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051916574Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"client":"0","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051926143Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"client":"1","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051936902Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"client":"2","":"downstairs","session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.05194663Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051955919Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"50cad5d6-fc20-465f-a69c-ff265d243452 is now active with session: dd74d965-c748-4986-af3e-8e065ffd7c67","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051965898Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051974627Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"session_id":"dd74d965-c748-4986-af3e-8e065ffd7c67"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051983696Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181} {"msg":"Activated sub_volume 50cad5d6-fc20-465f-a69c-ff265d243452","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051991695Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.051999744Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.052306109Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"task":"crutest"} 01/30 Write block 129 len 28672 data: 2 2 2 2 2 2 2 02/30 Read block 542 len 32768 03/30 Read block 2 len 28672 04/30 Write block 320 len 8192 data: 2 2 05/30 Read block 598 len 4096 06/30 Read block 200 len 32768 07/30 Read block 133 len 36864 08/30 Write block 28 len 36864 data: 2 2 2 2 2 2 2 3 3 09/30 Write block 297 len 8192 data: 2 2 10/30 Read block 208 len 16384 11/30 Write block 474 len 24576 data: 2 2 3 3 3 3 12/30 Read block 227 len 36864 13/30 Write block 2 len 28672 data: 2 3 3 2 2 2 2 14/30 Read block 136 len 24576 15/30 Write block 469 len 20480 data: 2 2 2 2 2 16/30 Write block 407 len 32768 data: 3 3 3 3 3 2 2 2 17/30 Read block 468 len 28672 18/30 Flush 19/30 Write block 476 len 16384 data: 4 4 4 4 20/30 Read block 234 len 16384 21/30 Read block 450 len 8192 22/30 Read block 546 len 4096 23/30 Write block 596 len 4096 data: 3 24/30 Write block 20 len 32768 data: 2 2 2 2 2 2 2 2 25/30 Read block 528 len 40960 26/30 Write block 269 len 8192 data: 2 2 27/30 Read block 45 len 20480 28/30 Write block 123 len 32768 data: 3 3 3 4 3 3 3 3 29/30 Write block 92 len 36864 data: 3 3 3 4 3 2 2 2 2 30/30 Write block 544 len 12288 data: 3 3 3 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Sent false 1001 Acked Read 8 Done Done Sent false 1002 Acked Read 7 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Read 1 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Write 9 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Read 4 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Read 5 Done Done Sent false 1027 Acked Write 8 Done Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 30 35 Done 28 27 0 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:41.06094218Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:41.0611174Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1181,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 11 22 22 21 F F F <--- 1 020-039 12 12 7 22 22 12 T T F <--- 2 040-059 7 7 7 12 12 12 F F F 3 060-079 7 7 7 12 12 12 F F F 4 080-099 10 10 10 17 17 17 T T F <--- 5 100-119 10 10 10 18 18 18 T T F <--- 6 120-139 12 12 7 22 22 12 T T F <--- 7 140-159 10 10 10 19 19 19 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 10 10 10 20 20 20 F F F 10 200-219 7 7 7 12 12 12 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 6 6 6 11 11 11 T T F <--- 14 280-299 12 12 6 22 22 11 F F F <--- 15 300-319 4 4 4 9 9 9 F F F 16 320-339 12 12 10 22 22 20 F F F <--- 17 340-359 10 10 10 20 20 20 F F F 18 360-379 10 10 10 20 20 20 F F F 19 380-399 10 10 10 20 20 20 F F F 20 400-419 12 12 9 22 22 16 F F F <--- 21 420-439 10 10 10 18 18 18 F F F 22 440-459 9 9 9 16 16 16 F F F 23 460-479 12 12 4 22 22 8 T T F <--- 24 480-499 6 6 6 11 11 11 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 8 8 8 13 13 13 T T F <--- 28 560-579 10 10 10 20 20 20 F F F 29 580-599 9 9 9 16 16 16 T T F <--- Max gen: 12, Max flush: 22 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 22 22 22 F F F 1 020-039 13 13 13 23 23 23 F F F 2 040-059 7 7 7 12 12 12 F F F 3 060-079 7 7 7 12 12 12 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 13 13 13 23 23 23 F F F 6 120-139 13 13 13 23 23 23 F F F 7 140-159 10 10 10 19 19 19 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 10 10 10 20 20 20 F F F 10 200-219 7 7 7 12 12 12 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 13 13 13 23 23 23 F F F 14 280-299 12 12 12 22 22 22 F F F 15 300-319 4 4 4 9 9 9 F F F 16 320-339 12 12 12 22 22 22 F F F 17 340-359 10 10 10 20 20 20 F F F 18 360-379 10 10 10 20 20 20 F F F 19 380-399 10 10 10 20 20 20 F F F 20 400-419 12 12 12 22 22 22 F F F 21 420-439 10 10 10 18 18 18 F F F 22 440-459 9 9 9 16 16 16 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 13 13 13 23 23 23 F F F 28 560-579 10 10 10 20 20 20 F F F 29 580-599 13 13 13 23 23 23 F F F Max gen: 13, Max flush: 23 Begin loop 7 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.308492708Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.316675133Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.316982107Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317006914Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317015563Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317027042Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189} {"msg":"Crucible 975db02e-ad37-4e3d-9f9c-c8ad88f55241 has session id: 6872989f-2724-44b8-b31d-7767f32f4cbf","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.31738842Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"Upstairs opts: Upstairs UUID: 975db02e-ad37-4e3d-9f9c-c8ad88f55241, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317408228Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"Crucible stats registered with UUID: 975db02e-ad37-4e3d-9f9c-c8ad88f55241","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317418186Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:45.317431855Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317591376Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317621083Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317632101Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317744168Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317759147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189} {"msg":"975db02e-ad37-4e3d-9f9c-c8ad88f55241 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317776255Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317785224Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"client":"0","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317794862Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"client":"1","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317804271Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"client":"2","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.31781376Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317823789Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317833838Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"ds_connection connected from Ok(127.0.0.1:57164)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317895001Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"ds_connection connected from Ok(127.0.0.1:48998)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317937206Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"ds_connection connected from Ok(127.0.0.1:61450)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.317996419Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.318575081Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"client":"0","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.318624106Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"client":"1","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.318649103Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"client":"2","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.318895424Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.318913582Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"[0]R flush_numbers[0..12]: [22, 23, 12, 12, 23, 23, 23, 19, 1, 20, 12, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.31892645Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"downstairs","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"[0]R generation[0..12]: [12, 13, 7, 7, 13, 13, 13, 10, 1, 10, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.318937169Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"downstairs","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.318948388Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"downstairs","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"[1]R flush_numbers[0..12]: [22, 23, 12, 12, 23, 23, 23, 19, 1, 20, 12, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.318958937Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"downstairs","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"[1]R generation[0..12]: [12, 13, 7, 7, 13, 13, 13, 10, 1, 10, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.318969315Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"downstairs","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.318980354Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"downstairs","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"[2]R flush_numbers[0..12]: [22, 23, 12, 12, 23, 23, 23, 19, 1, 20, 12, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.318990713Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"downstairs","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"[2]R generation[0..12]: [12, 13, 7, 7, 13, 13, 13, 10, 1, 10, 7, 1]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.31901439Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"downstairs","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319025599Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"downstairs","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319034968Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"downstairs","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319044637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"downstairs","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319054105Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"downstairs","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319063484Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"":"downstairs","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319073493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319082332Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"client":"0","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319092181Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"client":"1","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.31910197Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"client":"2","":"downstairs","session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319111979Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319121128Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"975db02e-ad37-4e3d-9f9c-c8ad88f55241 is now active with session: 6872989f-2724-44b8-b31d-7767f32f4cbf","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319130986Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319139735Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"session_id":"6872989f-2724-44b8-b31d-7767f32f4cbf"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319148804Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189} {"msg":"Activated sub_volume 975db02e-ad37-4e3d-9f9c-c8ad88f55241","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319156873Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319166322Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.319489655Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"task":"crutest"} 01/30 Write block 29 len 20480 data: 3 3 3 3 3 02/30 Flush 03/30 Write block 286 len 16384 data: 3 3 2 2 04/30 Write block 295 len 36864 data: 3 3 3 3 2 3 3 3 3 05/30 Write block 32 len 24576 data: 4 4 3 4 4 2 06/30 Write block 232 len 28672 data: 2 2 2 2 2 2 2 07/30 Read block 185 len 16384 08/30 Read block 216 len 40960 09/30 Write block 393 len 36864 data: 2 2 2 2 2 2 2 2 2 10/30 Read block 411 len 24576 11/30 Read block 101 len 24576 12/30 Read block 329 len 24576 13/30 Write block 144 len 40960 data: 2 2 3 3 3 3 4 3 3 4 14/30 Write block 398 len 16384 data: 3 3 3 3 15/30 Read block 3 len 28672 16/30 Write block 319 len 12288 data: 2 3 3 17/30 Write block 358 len 20480 data: 2 2 3 3 3 18/30 Read block 88 len 24576 19/30 Write block 164 len 16384 data: 2 2 2 2 20/30 Write block 358 len 24576 data: 3 3 4 4 4 3 21/30 Read block 438 len 36864 22/30 Write block 432 len 20480 data: 4 4 3 2 2 23/30 Write block 70 len 20480 data: 3 4 4 4 3 24/30 Write block 256 len 8192 data: 2 2 25/30 Read block 567 len 32768 26/30 Read block 132 len 8192 27/30 Write block 151 len 16384 data: 4 4 5 4 28/30 Write block 202 len 12288 data: 3 2 2 29/30 Read block 234 len 24576 30/30 Write block 150 len 20480 data: 5 5 5 6 5 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 4 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Write 7 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 2 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 22 1 24 Done 27 6 27 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:45.327618666Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:45.327790096Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1189,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 22 22 22 F F F 1 020-039 14 14 14 24 24 24 T T T 2 040-059 7 7 7 12 12 12 F F F 3 060-079 7 7 7 12 12 12 T F T <--- 4 080-099 13 13 13 23 23 23 F F F 5 100-119 13 13 13 23 23 23 F F F 6 120-139 13 13 13 23 23 23 F F F 7 140-159 10 10 10 19 19 19 T F T <--- 8 160-179 1 1 1 1 1 1 T F T <--- 9 180-199 10 10 10 20 20 20 F F F 10 200-219 7 7 7 12 12 12 T F T <--- 11 220-239 1 1 1 1 1 1 T T T 12 240-259 1 1 1 1 1 1 T F T <--- 13 260-279 13 13 13 23 23 23 F F F 14 280-299 12 12 12 22 22 22 T T T 15 300-319 4 4 4 9 9 9 T T T 16 320-339 12 12 12 22 22 22 T F T <--- 17 340-359 10 10 10 20 20 20 T F T <--- 18 360-379 10 10 10 20 20 20 T F T <--- 19 380-399 10 10 10 20 20 20 T F T <--- 20 400-419 12 12 12 22 22 22 T F T <--- 21 420-439 10 10 10 18 18 18 T F T <--- 22 440-459 9 9 9 16 16 16 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 13 13 13 23 23 23 F F F 28 560-579 10 10 10 20 20 20 F F F 29 580-599 13 13 13 23 23 23 F F F Max gen: 14, Max flush: 24 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 22 22 22 F F F 1 020-039 15 15 15 25 25 25 F F F 2 040-059 7 7 7 12 12 12 F F F 3 060-079 15 15 15 25 25 25 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 13 13 13 23 23 23 F F F 6 120-139 13 13 13 23 23 23 F F F 7 140-159 15 15 15 25 25 25 F F F 8 160-179 15 15 15 25 25 25 F F F 9 180-199 10 10 10 20 20 20 F F F 10 200-219 15 15 15 25 25 25 F F F 11 220-239 15 15 15 25 25 25 F F F 12 240-259 15 15 15 25 25 25 F F F 13 260-279 13 13 13 23 23 23 F F F 14 280-299 15 15 15 25 25 25 F F F 15 300-319 15 15 15 25 25 25 F F F 16 320-339 15 15 15 25 25 25 F F F 17 340-359 15 15 15 25 25 25 F F F 18 360-379 15 15 15 25 25 25 F F F 19 380-399 15 15 15 25 25 25 F F F 20 400-419 15 15 15 25 25 25 F F F 21 420-439 15 15 15 25 25 25 F F F 22 440-459 9 9 9 16 16 16 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 6 6 6 11 11 11 F F F 25 500-519 4 4 4 8 8 8 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 13 13 13 23 23 23 F F F 28 560-579 10 10 10 20 20 20 F F F 29 580-599 13 13 13 23 23 23 F F F Max gen: 15, Max flush: 25 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.580414501Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:49.587431242Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.587486276Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.594435505Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.594688995Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.594714872Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.594723421Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.59473484Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197} {"msg":"Crucible 61748133-015a-420d-ac5c-50d581dc2eae has session id: ec96742d-cc89-4c3a-814d-cc276e8828f7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595085829Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"Upstairs opts: Upstairs UUID: 61748133-015a-420d-ac5c-50d581dc2eae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595115865Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"Crucible stats registered with UUID: 61748133-015a-420d-ac5c-50d581dc2eae","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595127054Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:49.595136643Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.59525031Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595266098Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595276197Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595425869Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595437768Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197} {"msg":"61748133-015a-420d-ac5c-50d581dc2eae active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595470144Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595479473Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"client":"0","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595489212Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"client":"1","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595498741Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"client":"2","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595508369Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595518238Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595528137Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62885)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.59558523Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:45682)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595610917Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:38528)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.595722664Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596213497Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"client":"2","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596235605Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"client":"1","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596257582Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"client":"0","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596537329Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596602512Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"[0]R flush_numbers[0..12]: [22, 25, 12, 25, 23, 23, 23, 25, 25, 20, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596629259Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"downstairs","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"[0]R generation[0..12]: [12, 15, 7, 15, 13, 13, 13, 15, 15, 10, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596640787Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"downstairs","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596651766Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"downstairs","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"[1]R flush_numbers[0..12]: [22, 25, 12, 25, 23, 23, 23, 25, 25, 20, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596662185Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"downstairs","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"[1]R generation[0..12]: [12, 15, 7, 15, 13, 13, 13, 15, 15, 10, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596672604Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"downstairs","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596683462Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"downstairs","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"[2]R flush_numbers[0..12]: [22, 25, 12, 25, 23, 23, 23, 25, 25, 20, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596693861Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"downstairs","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"[2]R generation[0..12]: [12, 15, 7, 15, 13, 13, 13, 15, 15, 10, 15, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.59670427Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"downstairs","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596715339Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"downstairs","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596724797Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"downstairs","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596737196Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"downstairs","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596746605Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"downstairs","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596755924Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"":"downstairs","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596765873Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596774782Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"client":"0","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.59678467Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"client":"1","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596794239Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"client":"2","":"downstairs","session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596804078Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596813247Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"61748133-015a-420d-ac5c-50d581dc2eae is now active with session: ec96742d-cc89-4c3a-814d-cc276e8828f7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596823186Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596831905Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"session_id":"ec96742d-cc89-4c3a-814d-cc276e8828f7"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596841024Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197} {"msg":"Activated sub_volume 61748133-015a-420d-ac5c-50d581dc2eae","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596849073Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 1197} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.596862281Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.597115762Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"task":"crutest"} 01/30 Write block 209 len 4096 data: 3 02/30 Write block 510 len 40960 data: 2 2 2 2 2 3 3 3 3 3 03/30 Read block 266 len 28672 04/30 Write block 40 len 32768 data: 2 2 2 2 3 4 4 4 05/30 Write block 252 len 12288 data: 2 2 2 06/30 Write block 547 len 28672 data: 3 2 3 4 3 3 4 07/30 Flush 08/30 Write block 189 len 28672 data: 3 3 3 3 3 2 2 09/30 Write block 116 len 12288 data: 2 3 3 10/30 Write block 5 len 12288 data: 3 3 3 11/30 Read block 239 len 4096 12/30 Write block 442 len 36864 data: 3 3 2 2 2 2 3 3 3 13/30 Read block 500 len 12288 14/30 Write block 70 len 24576 data: 4 5 5 5 4 3 15/30 Read block 476 len 20480 16/30 Read block 497 len 20480 17/30 Write block 208 len 8192 data: 3 4 18/30 Write block 250 len 12288 data: 2 2 3 19/30 Read block 223 len 20480 20/30 Read block 97 len 12288 21/30 Read block 394 len 4096 22/30 Read block 175 len 32768 23/30 Write block 330 len 4096 data: 2 24/30 Write block 490 len 4096 data: 2 25/30 Write block 22 len 20480 data: 3 3 3 3 3 26/30 Write block 375 len 40960 data: 2 2 2 2 2 2 2 3 3 3 27/30 Write block 8 len 28672 data: 3 2 2 2 3 3 3 28/30 Write block 51 len 8192 data: 4 3 29/30 Read block 267 len 8192 30/30 Write block 44 len 24576 data: 4 5 5 5 3 3 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Read 7 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Write 3 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Read 1 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:49.604303513Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:49.604467124Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1197,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 22 22 22 F T T <--- 1 020-039 15 15 15 25 25 25 F T T <--- 2 040-059 7 16 16 12 26 26 F T T <--- 3 060-079 15 15 15 25 25 25 F T T <--- 4 080-099 13 13 13 23 23 23 F F F 5 100-119 13 13 13 23 23 23 F T T <--- 6 120-139 13 13 13 23 23 23 F F F 7 140-159 15 15 15 25 25 25 F F F 8 160-179 15 15 15 25 25 25 F F F 9 180-199 10 10 10 20 20 20 F T T <--- 10 200-219 15 16 16 25 26 26 T T T <--- 11 220-239 15 15 15 25 25 25 F F F 12 240-259 15 16 16 25 26 26 F T T <--- 13 260-279 13 13 13 23 23 23 F F F 14 280-299 15 15 15 25 25 25 F F F 15 300-319 15 15 15 25 25 25 F F F 16 320-339 15 15 15 25 25 25 F T T <--- 17 340-359 15 15 15 25 25 25 F F F 18 360-379 15 15 15 25 25 25 F T T <--- 19 380-399 15 15 15 25 25 25 F T T <--- 20 400-419 15 15 15 25 25 25 F F F 21 420-439 15 15 15 25 25 25 F F F 22 440-459 9 9 9 16 16 16 F T T <--- 23 460-479 13 13 13 23 23 23 F F F 24 480-499 6 6 6 11 11 11 F T T <--- 25 500-519 4 16 16 8 26 26 T F F <--- 26 520-539 11 11 11 21 21 21 F F F 27 540-559 13 16 16 23 26 26 F F F <--- 28 560-579 10 10 10 20 20 20 F F F 29 580-599 13 13 13 23 23 23 F F F Max gen: 16, Max flush: 26 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 F F F 1 020-039 17 17 17 27 27 27 F F F 2 040-059 17 17 17 27 27 27 F F F 3 060-079 17 17 17 27 27 27 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 17 17 17 27 27 27 F F F 6 120-139 13 13 13 23 23 23 F F F 7 140-159 15 15 15 25 25 25 F F F 8 160-179 15 15 15 25 25 25 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 17 17 17 27 27 27 F F F 11 220-239 15 15 15 25 25 25 F F F 12 240-259 17 17 17 27 27 27 F F F 13 260-279 13 13 13 23 23 23 F F F 14 280-299 15 15 15 25 25 25 F F F 15 300-319 15 15 15 25 25 25 F F F 16 320-339 17 17 17 27 27 27 F F F 17 340-359 15 15 15 25 25 25 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 17 17 17 27 27 27 F F F 20 400-419 15 15 15 25 25 25 F F F 21 420-439 15 15 15 25 25 25 F F F 22 440-459 17 17 17 27 27 27 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 17 17 17 27 27 27 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 16 16 16 26 26 26 F F F 28 560-579 10 10 10 20 20 20 F F F 29 580-599 13 13 13 23 23 23 F F F Max gen: 17, Max flush: 27 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.866257959Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.874530873Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.874822029Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.874856645Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.874865854Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.874877673Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205} {"msg":"Crucible a1864695-13a1-458d-a3e4-0493640e1ecf has session id: 74f2350e-9158-4aad-aadd-57493c2caca9","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875229881Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"Upstairs opts: Upstairs UUID: a1864695-13a1-458d-a3e4-0493640e1ecf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875248569Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"Crucible stats registered with UUID: a1864695-13a1-458d-a3e4-0493640e1ecf","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875258628Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:53.875267687Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.87540944Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875424569Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875434697Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875576881Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875589439Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205} {"msg":"a1864695-13a1-458d-a3e4-0493640e1ecf active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875623695Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875633204Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"client":"0","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875644863Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"client":"1","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875654502Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"client":"2","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875663971Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.87567416Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875684108Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"ds_connection connected from Ok(127.0.0.1:49140)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875767879Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"ds_connection connected from Ok(127.0.0.1:52010)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875793066Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"ds_connection connected from Ok(127.0.0.1:62130)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.875860888Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.87652369Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"client":"0","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876541238Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"client":"1","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876562926Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"client":"2","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876854772Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876864771Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"[0]R flush_numbers[0..12]: [27, 27, 27, 27, 23, 27, 23, 25, 25, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876889568Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"downstairs","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"[0]R generation[0..12]: [17, 17, 17, 17, 13, 17, 13, 15, 15, 17, 17, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876900786Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"downstairs","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876914785Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"downstairs","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"[1]R flush_numbers[0..12]: [27, 27, 27, 27, 23, 27, 23, 25, 25, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876925224Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"downstairs","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"[1]R generation[0..12]: [17, 17, 17, 17, 13, 17, 13, 15, 15, 17, 17, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876935572Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"downstairs","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876946521Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"downstairs","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"[2]R flush_numbers[0..12]: [27, 27, 27, 27, 23, 27, 23, 25, 25, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.87695694Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"downstairs","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"[2]R generation[0..12]: [17, 17, 17, 17, 13, 17, 13, 15, 15, 17, 17, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876967359Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"downstairs","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876978307Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"downstairs","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876987726Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"downstairs","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.876997555Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"downstairs","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.877006864Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"downstairs","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.877016453Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"":"downstairs","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.877026182Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.877035031Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"client":"0","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.87704488Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"client":"1","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.877055758Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"client":"2","":"downstairs","session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.877065627Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.877074756Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"a1864695-13a1-458d-a3e4-0493640e1ecf is now active with session: 74f2350e-9158-4aad-aadd-57493c2caca9","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.877084715Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.877093424Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"session_id":"74f2350e-9158-4aad-aadd-57493c2caca9"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.877102533Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205} {"msg":"Activated sub_volume a1864695-13a1-458d-a3e4-0493640e1ecf","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.877110902Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.877119141Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.877435874Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"task":"crutest"} 01/30 Write block 160 len 8192 data: 2 2 02/30 Write block 564 len 20480 data: 3 2 2 2 2 03/30 Read block 208 len 8192 04/30 Read block 211 len 24576 05/30 Read block 263 len 8192 06/30 Read block 5 len 32768 07/30 Read block 78 len 8192 08/30 Read block 391 len 8192 09/30 Write block 315 len 40960 data: 2 2 2 2 3 4 4 2 2 2 10/30 Read block 266 len 24576 11/30 Read block 531 len 28672 12/30 Write block 426 len 32768 data: 4 4 4 4 4 4 5 5 13/30 Flush 14/30 Flush 15/30 Write block 299 len 20480 data: 3 4 4 4 4 16/30 Write block 488 len 36864 data: 2 2 3 2 4 4 4 4 3 17/30 Write block 140 len 24576 data: 2 2 2 2 3 3 18/30 Write block 155 len 36864 data: 4 3 3 3 3 3 3 2 2 19/30 Write block 56 len 4096 data: 4 20/30 Read block 97 len 28672 21/30 Read block 67 len 28672 22/30 Read block 541 len 12288 23/30 Flush 24/30 Write block 328 len 40960 data: 2 2 3 2 2 3 2 2 2 3 25/30 Read block 549 len 4096 26/30 Read block 136 len 28672 27/30 Write block 159 len 40960 data: 4 4 4 3 3 3 3 3 3 2 28/30 Read block 374 len 12288 29/30 Write block 249 len 36864 data: 2 3 3 4 3 3 2 3 3 30/30 Write block 542 len 36864 data: 3 3 4 4 4 4 3 4 5 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Sent Done false 1001 Acked Write 5 Done Sent Done false 1002 Acked Read 2 Done Sent Done false 1003 Acked Read 6 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Read 8 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:53.886433044Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:53.886581726Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1205,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 F F F 1 020-039 17 17 17 27 27 27 F F F 2 040-059 18 17 18 30 27 30 F F F <--- 3 060-079 17 17 17 27 27 27 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 17 17 17 27 27 27 F F F 6 120-139 13 13 13 23 23 23 F F F 7 140-159 18 15 18 30 25 30 T F T <--- 8 160-179 18 15 18 30 25 30 T F T <--- 9 180-199 17 17 17 27 27 27 F F F 10 200-219 17 17 17 27 27 27 F F F 11 220-239 15 15 15 25 25 25 F F F 12 240-259 17 17 17 27 27 27 T F T <--- 13 260-279 13 13 13 23 23 23 F F F 14 280-299 18 15 18 30 25 30 F F F <--- 15 300-319 18 15 18 30 25 30 F F F <--- 16 320-339 18 17 18 28 27 28 T F T <--- 17 340-359 15 15 15 25 25 25 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 17 17 17 27 27 27 F F F 20 400-419 15 15 15 25 25 25 F F F 21 420-439 18 15 18 28 25 28 F F F <--- 22 440-459 17 17 17 27 27 27 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 18 17 18 30 27 30 F F F <--- 25 500-519 16 16 16 26 26 26 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 16 16 16 26 26 26 T F T <--- 28 560-579 18 10 18 28 20 28 F F F <--- 29 580-599 13 13 13 23 23 23 F F F Max gen: 18, Max flush: 30 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 F F F 1 020-039 17 17 17 27 27 27 F F F 2 040-059 18 18 18 30 30 30 F F F 3 060-079 17 17 17 27 27 27 F F F 4 080-099 13 13 13 23 23 23 F F F 5 100-119 17 17 17 27 27 27 F F F 6 120-139 13 13 13 23 23 23 F F F 7 140-159 19 19 19 31 31 31 F F F 8 160-179 19 19 19 31 31 31 F F F 9 180-199 17 17 17 27 27 27 F F F 10 200-219 17 17 17 27 27 27 F F F 11 220-239 15 15 15 25 25 25 F F F 12 240-259 19 19 19 31 31 31 F F F 13 260-279 13 13 13 23 23 23 F F F 14 280-299 18 18 18 30 30 30 F F F 15 300-319 18 18 18 30 30 30 F F F 16 320-339 19 19 19 31 31 31 F F F 17 340-359 15 15 15 25 25 25 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 17 17 17 27 27 27 F F F 20 400-419 15 15 15 25 25 25 F F F 21 420-439 18 18 18 28 28 28 F F F 22 440-459 17 17 17 27 27 27 F F F 23 460-479 13 13 13 23 23 23 F F F 24 480-499 18 18 18 30 30 30 F F F 25 500-519 16 16 16 26 26 26 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 18 18 18 28 28 28 F F F 29 580-599 13 13 13 23 23 23 F F F Max gen: 19, Max flush: 31 Begin loop 10 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.12851354Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:58.135629219Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.135689242Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.14281686Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143090208Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143117415Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143125844Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143137273Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213} {"msg":"Crucible 44120a11-725c-4818-88b0-19ea9690a95b has session id: 53e08a7f-e2f1-4152-a5f9-e1cae9b7e398","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143496761Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"Upstairs opts: Upstairs UUID: 44120a11-725c-4818-88b0-19ea9690a95b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143525208Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"Crucible stats registered with UUID: 44120a11-725c-4818-88b0-19ea9690a95b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143536136Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:58.143545355Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143689059Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143704797Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143714706Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143859679Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143875207Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213} {"msg":"44120a11-725c-4818-88b0-19ea9690a95b active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143910853Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143920532Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"client":"0","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.14393027Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"client":"1","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143940119Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"client":"2","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143949798Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143959717Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.143971556Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"ds_connection connected from Ok(127.0.0.1:44519)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.144043197Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"ds_connection connected from Ok(127.0.0.1:33766)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.144067934Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"ds_connection connected from Ok(127.0.0.1:55918)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.144138136Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.144739496Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"client":"1","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.144755174Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"client":"2","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.144897078Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"client":"0","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145056799Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145074287Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"[0]R flush_numbers[0..12]: [27, 27, 30, 27, 23, 27, 23, 31, 31, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145172135Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"downstairs","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"[0]R generation[0..12]: [17, 17, 18, 17, 13, 17, 13, 19, 19, 17, 17, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145183324Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"downstairs","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145194313Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"downstairs","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"[1]R flush_numbers[0..12]: [27, 27, 30, 27, 23, 27, 23, 31, 31, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145204662Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"downstairs","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"[1]R generation[0..12]: [17, 17, 18, 17, 13, 17, 13, 19, 19, 17, 17, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.14521513Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"downstairs","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145228139Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"downstairs","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"[2]R flush_numbers[0..12]: [27, 27, 30, 27, 23, 27, 23, 31, 31, 27, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145238688Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"downstairs","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"[2]R generation[0..12]: [17, 17, 18, 17, 13, 17, 13, 19, 19, 17, 17, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145249016Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"downstairs","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145259755Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"downstairs","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145269164Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"downstairs","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145278763Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"downstairs","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145288072Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"downstairs","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145297361Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"":"downstairs","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.14530705Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145315769Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"client":"0","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145325478Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"client":"1","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145335156Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"client":"2","":"downstairs","session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145345075Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145354194Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"44120a11-725c-4818-88b0-19ea9690a95b is now active with session: 53e08a7f-e2f1-4152-a5f9-e1cae9b7e398","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-09-16T04:39:58.145364113Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145377241Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"session_id":"53e08a7f-e2f1-4152-a5f9-e1cae9b7e398"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.14538768Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213} {"msg":"Activated sub_volume 44120a11-725c-4818-88b0-19ea9690a95b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145395589Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145403738Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.145625483Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"task":"crutest"} 01/30 Write block 270 len 40960 data: 3 2 2 3 3 3 3 3 3 3 02/30 Read block 210 len 32768 03/30 Read block 498 len 20480 04/30 Read block 466 len 40960 05/30 Read block 349 len 28672 06/30 Read block 393 len 32768 07/30 Read block 107 len 24576 08/30 Write block 439 len 40960 data: 2 2 3 4 4 3 3 3 3 4 09/30 Read block 402 len 20480 10/30 Read block 201 len 20480 11/30 Write block 189 len 40960 data: 4 4 4 4 4 3 3 2 2 4 12/30 Write block 342 len 36864 data: 3 3 3 3 3 2 2 2 2 13/30 Flush 14/30 Write block 492 len 36864 data: 5 5 5 5 4 2 2 2 2 15/30 Read block 269 len 20480 16/30 Read block 394 len 12288 17/30 Write block 478 len 32768 data: 5 5 3 3 3 2 2 2 18/30 Read block 559 len 16384 19/30 Write block 321 len 4096 data: 5 20/30 Write block 561 len 16384 data: 3 3 3 4 21/30 Read block 26 len 24576 22/30 Read block 419 len 12288 23/30 Flush 24/30 Flush 25/30 Write block 85 len 20480 data: 3 3 4 3 3 26/30 Read block 256 len 16384 27/30 Write block 185 len 20480 data: 2 2 2 3 5 28/30 Read block 246 len 28672 29/30 Read block 238 len 32768 30/30 Read block 300 len 24576 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 8 Sent Done Done false 1002 Acked Read 5 Sent Done Done false 1003 Acked Read 10 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 6 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Write 10 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:39:58.155805654Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:39:58.155971015Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1213,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 F F F 1 020-039 17 17 17 27 27 27 F F F 2 040-059 18 18 18 30 30 30 F F F 3 060-079 17 17 17 27 27 27 F F F 4 080-099 13 13 13 23 23 23 F T T <--- 5 100-119 17 17 17 27 27 27 F F F 6 120-139 13 13 13 23 23 23 F F F 7 140-159 19 19 19 31 31 31 F F F 8 160-179 19 19 19 31 31 31 F F F 9 180-199 17 20 20 27 32 32 F T T <--- 10 200-219 17 17 17 27 27 27 F F F 11 220-239 15 15 15 25 25 25 F F F 12 240-259 19 19 19 31 31 31 F F F 13 260-279 13 20 20 23 32 32 T F F <--- 14 280-299 18 18 18 30 30 30 F F F 15 300-319 18 18 18 30 30 30 F F F 16 320-339 19 20 20 31 33 33 F F F <--- 17 340-359 15 20 20 25 32 32 F F F <--- 18 360-379 17 17 17 27 27 27 F F F 19 380-399 17 17 17 27 27 27 F F F 20 400-419 15 15 15 25 25 25 F F F 21 420-439 18 20 20 28 32 32 F F F <--- 22 440-459 17 20 20 27 32 32 F F F <--- 23 460-479 13 20 20 23 33 33 F F F <--- 24 480-499 18 20 20 30 33 33 F F F <--- 25 500-519 16 20 20 26 33 33 F F F <--- 26 520-539 11 11 11 21 21 21 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 18 20 20 28 33 33 F F F <--- 29 580-599 13 13 13 23 23 23 F F F Max gen: 20, Max flush: 33 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 27 27 27 F F F 1 020-039 17 17 17 27 27 27 F F F 2 040-059 18 18 18 30 30 30 F F F 3 060-079 17 17 17 27 27 27 F F F 4 080-099 21 21 21 34 34 34 F F F 5 100-119 17 17 17 27 27 27 F F F 6 120-139 13 13 13 23 23 23 F F F 7 140-159 19 19 19 31 31 31 F F F 8 160-179 19 19 19 31 31 31 F F F 9 180-199 21 21 21 34 34 34 F F F 10 200-219 17 17 17 27 27 27 F F F 11 220-239 15 15 15 25 25 25 F F F 12 240-259 19 19 19 31 31 31 F F F 13 260-279 20 20 20 32 32 32 F F F 14 280-299 18 18 18 30 30 30 F F F 15 300-319 18 18 18 30 30 30 F F F 16 320-339 20 20 20 33 33 33 F F F 17 340-359 20 20 20 32 32 32 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 17 17 17 27 27 27 F F F 20 400-419 15 15 15 25 25 25 F F F 21 420-439 20 20 20 32 32 32 F F F 22 440-459 20 20 20 32 32 32 F F F 23 460-479 20 20 20 33 33 33 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 11 11 11 21 21 21 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 20 20 20 33 33 33 F F F 29 580-599 13 13 13 23 23 23 F F F Max gen: 21, Max flush: 34 Begin loop 11 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.400692153Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.409055637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.409333964Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.409361991Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.40937074Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.409382239Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221} {"msg":"Crucible 494ec629-0a2c-416b-86dc-ec8b0d93e084 has session id: 27abfc91-4ffa-4ca5-bc2e-b018b1a6481e","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.409701931Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"Upstairs opts: Upstairs UUID: 494ec629-0a2c-416b-86dc-ec8b0d93e084, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.409721059Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"Crucible stats registered with UUID: 494ec629-0a2c-416b-86dc-ec8b0d93e084","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.409730868Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:02.409739867Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.4098849Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.409917066Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.409928525Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410027423Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410046381Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221} {"msg":"494ec629-0a2c-416b-86dc-ec8b0d93e084 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410069678Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410079057Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"client":"0","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410088816Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"client":"1","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410098425Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"client":"2","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410107904Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410117923Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410127702Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33975)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410187935Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37286)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410198833Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60067)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410209122Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410850437Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"client":"0","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410886163Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"client":"1","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.410931718Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"client":"2","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411130515Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411154842Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"[0]R flush_numbers[0..12]: [27, 27, 30, 27, 34, 27, 23, 31, 31, 34, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411197657Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"downstairs","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"[0]R generation[0..12]: [17, 17, 18, 17, 21, 17, 13, 19, 19, 21, 17, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411208625Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"downstairs","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411219704Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"downstairs","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"[1]R flush_numbers[0..12]: [27, 27, 30, 27, 34, 27, 23, 31, 31, 34, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411230073Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"downstairs","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"[1]R generation[0..12]: [17, 17, 18, 17, 21, 17, 13, 19, 19, 21, 17, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411240442Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"downstairs","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.4112513Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"downstairs","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"[2]R flush_numbers[0..12]: [27, 27, 30, 27, 34, 27, 23, 31, 31, 34, 27, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411261739Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"downstairs","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"[2]R generation[0..12]: [17, 17, 18, 17, 21, 17, 13, 19, 19, 21, 17, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411272068Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"downstairs","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411283097Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"downstairs","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411292456Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"downstairs","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411306744Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"downstairs","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"Next flush: 35","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411317873Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"downstairs","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411327152Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"":"downstairs","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.41133711Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411346149Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"client":"0","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411355618Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"client":"1","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411365147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"client":"2","":"downstairs","session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411375066Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411384165Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"494ec629-0a2c-416b-86dc-ec8b0d93e084 is now active with session: 27abfc91-4ffa-4ca5-bc2e-b018b1a6481e","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411394104Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411402743Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"session_id":"27abfc91-4ffa-4ca5-bc2e-b018b1a6481e"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411411782Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221} {"msg":"Activated sub_volume 494ec629-0a2c-416b-86dc-ec8b0d93e084","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411419691Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-09-16T04:40:02.41142785Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.411671941Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"task":"crutest"} 01/30 Write block 99 len 24576 data: 3 3 2 3 3 3 02/30 Read block 48 len 20480 03/30 Read block 197 len 4096 04/30 Read block 166 len 20480 05/30 Write block 246 len 24576 data: 2 2 2 3 4 4 06/30 Write block 520 len 32768 data: 3 4 2 2 2 2 2 2 07/30 Write block 4 len 4096 data: 4 08/30 Read block 252 len 20480 09/30 Read block 500 len 16384 10/30 Flush 11/30 Write block 316 len 20480 data: 3 3 3 4 5 12/30 Write block 251 len 24576 data: 5 5 4 4 3 4 13/30 Flush 14/30 Write block 218 len 4096 data: 2 15/30 Write block 456 len 16384 data: 3 3 3 4 16/30 Read block 433 len 20480 17/30 Read block 557 len 28672 18/30 Write block 571 len 36864 data: 3 4 4 4 5 3 2 2 3 19/30 Read block 483 len 36864 20/30 Read block 504 len 12288 21/30 Read block 208 len 32768 22/30 Read block 130 len 16384 23/30 Flush 24/30 Write block 592 len 8192 data: 3 3 25/30 Write block 82 len 36864 data: 5 5 4 4 4 5 4 4 3 26/30 Read block 504 len 20480 27/30 Read block 207 len 28672 28/30 Read block 51 len 36864 29/30 Read block 315 len 24576 30/30 Write block 577 len 36864 data: 3 3 4 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 1 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Write 8 Done Done Sent false 1006 Acked Write 1 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Read 4 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:02.420343799Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:02.420523758Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1221,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 17 35 35 27 F F F <--- 1 020-039 17 17 17 27 27 27 F F F 2 040-059 18 18 18 30 30 30 F F F 3 060-079 17 17 17 27 27 27 F F F 4 080-099 22 22 21 35 35 34 T T T <--- 5 100-119 22 22 17 35 35 27 F F T <--- 6 120-139 13 13 13 23 23 23 F F F 7 140-159 19 19 19 31 31 31 F F F 8 160-179 19 19 19 31 31 31 F F F 9 180-199 21 21 21 34 34 34 F F F 10 200-219 22 22 17 37 37 27 F F F <--- 11 220-239 15 15 15 25 25 25 F F F 12 240-259 22 22 19 36 36 31 F F F <--- 13 260-279 20 20 20 32 32 32 F F F 14 280-299 18 18 18 30 30 30 F F F 15 300-319 22 22 18 36 36 30 F F F <--- 16 320-339 22 22 20 36 36 33 F F F <--- 17 340-359 20 20 20 32 32 32 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 17 17 17 27 27 27 F F F 20 400-419 15 15 15 25 25 25 F F F 21 420-439 20 20 20 32 32 32 F F F 22 440-459 22 22 20 37 37 32 F F F <--- 23 460-479 20 20 20 33 33 33 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 22 22 11 35 35 21 F F F <--- 27 540-559 19 19 19 31 31 31 F F F 28 560-579 22 22 20 37 37 33 T T F <--- 29 580-599 13 13 13 23 23 23 T T F <--- Max gen: 22, Max flush: 37 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 35 35 35 F F F 1 020-039 17 17 17 27 27 27 F F F 2 040-059 18 18 18 30 30 30 F F F 3 060-079 17 17 17 27 27 27 F F F 4 080-099 23 23 23 38 38 38 F F F 5 100-119 22 22 22 35 35 35 F F F 6 120-139 13 13 13 23 23 23 F F F 7 140-159 19 19 19 31 31 31 F F F 8 160-179 19 19 19 31 31 31 F F F 9 180-199 21 21 21 34 34 34 F F F 10 200-219 22 22 22 37 37 37 F F F 11 220-239 15 15 15 25 25 25 F F F 12 240-259 22 22 22 36 36 36 F F F 13 260-279 20 20 20 32 32 32 F F F 14 280-299 18 18 18 30 30 30 F F F 15 300-319 22 22 22 36 36 36 F F F 16 320-339 22 22 22 36 36 36 F F F 17 340-359 20 20 20 32 32 32 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 17 17 17 27 27 27 F F F 20 400-419 15 15 15 25 25 25 F F F 21 420-439 20 20 20 32 32 32 F F F 22 440-459 22 22 22 37 37 37 F F F 23 460-479 20 20 20 33 33 33 F F F 24 480-499 20 20 20 33 33 33 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 22 22 22 35 35 35 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 23 23 23 38 38 38 F F F 29 580-599 23 23 23 38 38 38 F F F Max gen: 23, Max flush: 38 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.65432879Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.662501606Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.662787723Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.662821989Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.662833748Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.662845386Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229} {"msg":"Crucible 1edf0fcb-b927-4792-b5c9-697e7fdde456 has session id: b9738948-ff7c-4b6e-a32e-2626da5ccdba","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663122874Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"Upstairs opts: Upstairs UUID: 1edf0fcb-b927-4792-b5c9-697e7fdde456, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663148571Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"Crucible stats registered with UUID: 1edf0fcb-b927-4792-b5c9-697e7fdde456","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.66315911Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:06.663173528Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663318391Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663333759Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663343918Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663468813Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663492731Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229} {"msg":"1edf0fcb-b927-4792-b5c9-697e7fdde456 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.66350223Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663511029Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"client":"0","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663520677Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"client":"1","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663532396Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"client":"2","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663542115Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663552064Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663562053Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"ds_connection connected from Ok(127.0.0.1:52506)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663602278Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"ds_connection connected from Ok(127.0.0.1:49481)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663618816Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"ds_connection connected from Ok(127.0.0.1:43244)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.663632564Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664263761Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"client":"2","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664286778Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"client":"0","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664324084Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"client":"1","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664598682Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664645756Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"[0]R flush_numbers[0..12]: [35, 27, 30, 27, 38, 35, 23, 31, 31, 34, 37, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664713548Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"downstairs","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"[0]R generation[0..12]: [22, 17, 18, 17, 23, 22, 13, 19, 19, 21, 22, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664724697Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"downstairs","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664735716Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"downstairs","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"[1]R flush_numbers[0..12]: [35, 27, 30, 27, 38, 35, 23, 31, 31, 34, 37, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664748684Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"downstairs","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"[1]R generation[0..12]: [22, 17, 18, 17, 23, 22, 13, 19, 19, 21, 22, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664759233Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"downstairs","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664770232Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"downstairs","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"[2]R flush_numbers[0..12]: [35, 27, 30, 27, 38, 35, 23, 31, 31, 34, 37, 25]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.66478069Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"downstairs","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"[2]R generation[0..12]: [22, 17, 18, 17, 23, 22, 13, 19, 19, 21, 22, 15]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664791579Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"downstairs","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664802778Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"downstairs","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664812257Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"downstairs","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664821875Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"downstairs","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664831184Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"downstairs","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664840603Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"":"downstairs","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664850422Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664859291Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"client":"0","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.66486886Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"client":"1","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664878479Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"client":"2","":"downstairs","session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664888648Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664899006Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"1edf0fcb-b927-4792-b5c9-697e7fdde456 is now active with session: b9738948-ff7c-4b6e-a32e-2626da5ccdba","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664908985Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664917714Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"session_id":"b9738948-ff7c-4b6e-a32e-2626da5ccdba"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664926913Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229} {"msg":"Activated sub_volume 1edf0fcb-b927-4792-b5c9-697e7fdde456","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664934852Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.664942971Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.66521383Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"task":"crutest"} 01/30 Write block 266 len 40960 data: 2 2 2 3 4 3 3 4 4 4 02/30 Read block 360 len 24576 03/30 Read block 333 len 36864 04/30 Read block 220 len 12288 05/30 Write block 462 len 12288 data: 3 3 3 06/30 Read block 104 len 12288 07/30 Read block 467 len 12288 08/30 Write block 486 len 12288 data: 2 2 3 09/30 Write block 267 len 40960 data: 3 3 4 5 4 4 5 5 5 4 10/30 Read block 18 len 24576 11/30 Read block 243 len 12288 12/30 Read block 545 len 36864 13/30 Write block 583 len 4096 data: 4 14/30 Read block 382 len 8192 15/30 Flush 16/30 Write block 328 len 4096 data: 3 17/30 Write block 31 len 16384 data: 4 5 5 4 18/30 Read block 523 len 40960 19/30 Write block 262 len 24576 data: 2 2 2 2 3 4 20/30 Write block 98 len 16384 data: 3 4 4 3 21/30 Write block 235 len 20480 data: 3 3 3 3 2 22/30 Write block 276 len 16384 data: 5 4 4 4 23/30 Write block 203 len 28672 data: 3 3 2 2 2 4 5 24/30 Write block 134 len 20480 data: 3 3 2 2 2 25/30 Flush 26/30 Write block 333 len 32768 data: 4 3 3 3 4 3 3 3 27/30 Write block 181 len 20480 data: 2 2 2 2 3 28/30 Read block 238 len 36864 29/30 Write block 86 len 20480 data: 5 6 5 5 4 30/30 Read block 296 len 28672 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 5 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 1 27 Done 30 4 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:06.673494303Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:06.673669443Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1229,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 35 35 35 F F F 1 020-039 24 17 24 40 27 40 F F F <--- 2 040-059 18 18 18 30 30 30 F F F 3 060-079 17 17 17 27 27 27 F F F 4 080-099 24 23 24 40 38 40 T F T <--- 5 100-119 24 22 24 40 35 40 F F F <--- 6 120-139 24 13 24 40 23 40 F F F <--- 7 140-159 19 19 19 31 31 31 F F F 8 160-179 19 19 19 31 31 31 F F F 9 180-199 21 21 21 34 34 34 T F T <--- 10 200-219 24 22 24 40 37 40 F F F <--- 11 220-239 24 15 24 40 25 40 F F F <--- 12 240-259 22 22 22 36 36 36 F F F 13 260-279 24 20 24 40 32 40 F T F <--- 14 280-299 18 18 18 30 30 30 F F F 15 300-319 22 22 22 36 36 36 F F F 16 320-339 24 22 24 40 36 40 T F T <--- 17 340-359 20 20 20 32 32 32 T F T <--- 18 360-379 17 17 17 27 27 27 F F F 19 380-399 17 17 17 27 27 27 F F F 20 400-419 15 15 15 25 25 25 F F F 21 420-439 20 20 20 32 32 32 F F F 22 440-459 22 22 22 37 37 37 F F F 23 460-479 24 20 24 39 33 39 F F F <--- 24 480-499 24 20 24 39 33 39 F F F <--- 25 500-519 20 20 20 33 33 33 F F F 26 520-539 22 22 22 35 35 35 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 23 23 23 38 38 38 F F F 29 580-599 24 23 24 39 38 39 F F F <--- Max gen: 24, Max flush: 40 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 22 22 35 35 35 F F F 1 020-039 24 24 24 40 40 40 F F F 2 040-059 18 18 18 30 30 30 F F F 3 060-079 17 17 17 27 27 27 F F F 4 080-099 25 25 25 41 41 41 F F F 5 100-119 24 24 24 40 40 40 F F F 6 120-139 24 24 24 40 40 40 F F F 7 140-159 19 19 19 31 31 31 F F F 8 160-179 19 19 19 31 31 31 F F F 9 180-199 25 25 25 41 41 41 F F F 10 200-219 24 24 24 40 40 40 F F F 11 220-239 24 24 24 40 40 40 F F F 12 240-259 22 22 22 36 36 36 F F F 13 260-279 24 24 24 40 40 40 F F F 14 280-299 18 18 18 30 30 30 F F F 15 300-319 22 22 22 36 36 36 F F F 16 320-339 25 25 25 41 41 41 F F F 17 340-359 25 25 25 41 41 41 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 17 17 17 27 27 27 F F F 20 400-419 15 15 15 25 25 25 F F F 21 420-439 20 20 20 32 32 32 F F F 22 440-459 22 22 22 37 37 37 F F F 23 460-479 24 24 24 39 39 39 F F F 24 480-499 24 24 24 39 39 39 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 22 22 22 35 35 35 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 23 23 23 38 38 38 F F F 29 580-599 24 24 24 39 39 39 F F F Max gen: 25, Max flush: 41 Begin loop 13 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.917763493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:10.925066111Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.925127884Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.932306556Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.932562236Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.932586173Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.932594832Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.932606421Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237} {"msg":"Crucible 19af65ea-cd3b-4e52-9559-246dfadcc5c9 has session id: 7e27bc3b-0e24-4022-b847-c7fd638bf68f","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933005694Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"Upstairs opts: Upstairs UUID: 19af65ea-cd3b-4e52-9559-246dfadcc5c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933029062Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"Crucible stats registered with UUID: 19af65ea-cd3b-4e52-9559-246dfadcc5c9","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.93303885Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:10.933048029Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933194262Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933223099Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933233968Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933390909Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933409927Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237} {"msg":"19af65ea-cd3b-4e52-9559-246dfadcc5c9 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933427865Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933436864Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"client":"0","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933446533Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"client":"1","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933456232Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"client":"2","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933465661Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933475539Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933485558Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34233)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933533053Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44081)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.93355456Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55814)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.933565279Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934163109Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"client":"1","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934256658Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"client":"2","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934276816Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"client":"0","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934493521Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934512958Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"[0]R flush_numbers[0..12]: [35, 40, 30, 27, 41, 40, 40, 31, 31, 41, 40, 40]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934577221Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"downstairs","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"[0]R generation[0..12]: [22, 24, 18, 17, 25, 24, 24, 19, 19, 25, 24, 24]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934588579Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"downstairs","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934599778Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"downstairs","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"[1]R flush_numbers[0..12]: [35, 40, 30, 27, 41, 40, 40, 31, 31, 41, 40, 40]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934610247Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"downstairs","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"[1]R generation[0..12]: [22, 24, 18, 17, 25, 24, 24, 19, 19, 25, 24, 24]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934620656Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"downstairs","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934631754Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"downstairs","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"[2]R flush_numbers[0..12]: [35, 40, 30, 27, 41, 40, 40, 31, 31, 41, 40, 40]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934642183Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"downstairs","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"[2]R generation[0..12]: [22, 24, 18, 17, 25, 24, 24, 19, 19, 25, 24, 24]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934655042Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"downstairs","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.9346659Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"downstairs","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934675659Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"downstairs","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934685368Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"downstairs","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934694867Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"downstairs","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934704246Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"":"downstairs","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934713895Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934722684Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"client":"0","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934732353Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"client":"1","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934742022Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"client":"2","":"downstairs","session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.93475163Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934760829Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"19af65ea-cd3b-4e52-9559-246dfadcc5c9 is now active with session: 7e27bc3b-0e24-4022-b847-c7fd638bf68f","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934770818Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934779557Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"session_id":"7e27bc3b-0e24-4022-b847-c7fd638bf68f"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934788546Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237} {"msg":"Activated sub_volume 19af65ea-cd3b-4e52-9559-246dfadcc5c9","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934796735Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.934806444Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.935071313Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"task":"crutest"} 01/30 Write block 346 len 4096 data: 4 02/30 Read block 312 len 36864 03/30 Read block 247 len 28672 04/30 Write block 217 len 28672 data: 2 3 2 2 2 2 2 05/30 Read block 287 len 12288 06/30 Read block 422 len 32768 07/30 Write block 304 len 8192 data: 3 3 08/30 Flush 09/30 Write block 10 len 20480 data: 3 3 4 4 4 10/30 Flush 11/30 Flush 12/30 Write block 26 len 24576 data: 4 3 3 4 4 5 13/30 Write block 10 len 8192 data: 4 4 14/30 Read block 30 len 40960 15/30 Write block 263 len 8192 data: 3 3 16/30 Read block 204 len 4096 17/30 Read block 467 len 32768 18/30 Read block 84 len 20480 19/30 Flush 20/30 Write block 380 len 28672 data: 3 3 4 4 4 3 3 21/30 Read block 151 len 32768 22/30 Read block 246 len 20480 23/30 Read block 57 len 36864 24/30 Read block 149 len 4096 25/30 Write block 260 len 32768 data: 2 2 3 4 4 3 4 5 26/30 Read block 587 len 8192 27/30 Write block 89 len 32768 data: 6 5 3 4 4 4 5 4 28/30 Read block 560 len 4096 29/30 Read block 377 len 12288 30/30 Write block 473 len 20480 data: 3 3 3 5 5 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 7 Sent Done Done false 1004 Acked Read 3 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Write 2 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 1 29 Done 3 29 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:10.944379657Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:10.944547517Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1237,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 22 26 26 35 45 45 F F F <--- 1 020-039 24 26 26 40 45 45 F F F <--- 2 040-059 18 18 18 30 30 30 F F F 3 060-079 17 17 17 27 27 27 F F F 4 080-099 25 25 25 41 41 41 F T T <--- 5 100-119 24 24 24 40 40 40 F F F 6 120-139 24 24 24 40 40 40 F F F 7 140-159 19 19 19 31 31 31 F F F 8 160-179 19 19 19 31 31 31 F F F 9 180-199 25 25 25 41 41 41 F F F 10 200-219 24 26 26 40 42 42 F F F <--- 11 220-239 24 26 26 40 42 42 F F F <--- 12 240-259 22 22 22 36 36 36 F F F 13 260-279 24 26 26 40 45 45 F T T <--- 14 280-299 18 18 18 30 30 30 F F F 15 300-319 22 26 26 36 42 42 F F F <--- 16 320-339 25 25 25 41 41 41 F F F 17 340-359 25 26 26 41 42 42 T F F <--- 18 360-379 17 17 17 27 27 27 F F F 19 380-399 17 17 17 27 27 27 F T T <--- 20 400-419 15 15 15 25 25 25 F F F 21 420-439 20 20 20 32 32 32 F F F 22 440-459 22 22 22 37 37 37 F F F 23 460-479 24 24 24 39 39 39 F T T <--- 24 480-499 24 24 24 39 39 39 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 22 22 22 35 35 35 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 23 23 23 38 38 38 F F F 29 580-599 24 24 24 39 39 39 F F F Max gen: 26, Max flush: 45 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 45 45 45 F F F 1 020-039 26 26 26 45 45 45 F F F 2 040-059 18 18 18 30 30 30 F F F 3 060-079 17 17 17 27 27 27 F F F 4 080-099 27 27 27 46 46 46 F F F 5 100-119 24 24 24 40 40 40 F F F 6 120-139 24 24 24 40 40 40 F F F 7 140-159 19 19 19 31 31 31 F F F 8 160-179 19 19 19 31 31 31 F F F 9 180-199 25 25 25 41 41 41 F F F 10 200-219 26 26 26 42 42 42 F F F 11 220-239 26 26 26 42 42 42 F F F 12 240-259 22 22 22 36 36 36 F F F 13 260-279 27 27 27 46 46 46 F F F 14 280-299 18 18 18 30 30 30 F F F 15 300-319 26 26 26 42 42 42 F F F 16 320-339 25 25 25 41 41 41 F F F 17 340-359 26 26 26 42 42 42 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 27 27 27 46 46 46 F F F 20 400-419 15 15 15 25 25 25 F F F 21 420-439 20 20 20 32 32 32 F F F 22 440-459 22 22 22 37 37 37 F F F 23 460-479 27 27 27 46 46 46 F F F 24 480-499 24 24 24 39 39 39 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 22 22 22 35 35 35 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 23 23 23 38 38 38 F F F 29 580-599 24 24 24 39 39 39 F F F Max gen: 27, Max flush: 46 Begin loop 14 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.188443551Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.196771069Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197079933Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.19710497Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197114029Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197125668Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245} {"msg":"Crucible b98c9aab-b834-4763-a662-5bfd05d5cbdc has session id: 5261ed28-8b80-45a5-9339-00a91780bfd6","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197462899Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"Upstairs opts: Upstairs UUID: b98c9aab-b834-4763-a662-5bfd05d5cbdc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197515912Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"Crucible stats registered with UUID: b98c9aab-b834-4763-a662-5bfd05d5cbdc","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197526761Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:15.19753577Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197683813Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197716089Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197727248Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197856473Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197871351Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245} {"msg":"b98c9aab-b834-4763-a662-5bfd05d5cbdc active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197894528Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197903897Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"client":"0","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197913626Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"client":"1","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197923365Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"client":"2","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197932934Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197942833Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.197952781Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:36770)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.198041661Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:52311)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.198066668Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61978)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.198077757Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.198803232Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"client":"0","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.198842938Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"client":"1","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.198872214Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"client":"2","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199149312Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199174419Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"[0]R flush_numbers[0..12]: [45, 45, 30, 27, 46, 40, 40, 31, 31, 41, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199184868Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"downstairs","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"[0]R generation[0..12]: [26, 26, 18, 17, 27, 24, 24, 19, 19, 25, 26, 26]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199195576Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"downstairs","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199206765Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"downstairs","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"[1]R flush_numbers[0..12]: [45, 45, 30, 27, 46, 40, 40, 31, 31, 41, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199217434Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"downstairs","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"[1]R generation[0..12]: [26, 26, 18, 17, 27, 24, 24, 19, 19, 25, 26, 26]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199227873Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"downstairs","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199238761Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"downstairs","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"[2]R flush_numbers[0..12]: [45, 45, 30, 27, 46, 40, 40, 31, 31, 41, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.19924918Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"downstairs","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"[2]R generation[0..12]: [26, 26, 18, 17, 27, 24, 24, 19, 19, 25, 26, 26]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199259619Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"downstairs","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199270808Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"downstairs","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199280126Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"downstairs","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199289705Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"downstairs","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199299044Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"downstairs","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199308433Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"":"downstairs","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199319792Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199328581Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"client":"0","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.19933854Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"client":"1","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199348259Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"client":"2","":"downstairs","session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199358067Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199367216Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"b98c9aab-b834-4763-a662-5bfd05d5cbdc is now active with session: 5261ed28-8b80-45a5-9339-00a91780bfd6","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199377005Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199385684Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"session_id":"5261ed28-8b80-45a5-9339-00a91780bfd6"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199394843Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245} {"msg":"Activated sub_volume b98c9aab-b834-4763-a662-5bfd05d5cbdc","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199402832Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199411051Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.199731224Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"task":"crutest"} 01/30 Write block 398 len 20480 data: 4 4 4 4 2 02/30 Write block 427 len 4096 data: 5 03/30 Flush 04/30 Write block 183 len 4096 data: 3 05/30 Write block 88 len 8192 data: 6 7 06/30 Read block 480 len 4096 07/30 Write block 28 len 32768 data: 4 5 5 6 6 6 5 5 08/30 Read block 157 len 24576 09/30 Read block 398 len 36864 10/30 Read block 57 len 8192 11/30 Read block 280 len 20480 12/30 Read block 78 len 16384 13/30 Read block 410 len 28672 14/30 Write block 242 len 28672 data: 2 2 2 2 3 3 3 15/30 Write block 239 len 24576 data: 3 2 2 3 3 3 16/30 Read block 0 len 12288 17/30 Read block 305 len 40960 18/30 Write block 70 len 20480 data: 5 6 6 6 5 19/30 Read block 466 len 8192 20/30 Read block 595 len 8192 21/30 Read block 52 len 24576 22/30 Write block 299 len 4096 data: 4 23/30 Read block 163 len 40960 24/30 Read block 370 len 4096 25/30 Write block 176 len 12288 data: 2 2 2 26/30 Write block 244 len 12288 data: 4 3 4 27/30 Read block 573 len 4096 28/30 Read block 381 len 20480 29/30 Read block 212 len 8192 30/30 Write block 585 len 32768 data: 4 3 3 3 4 4 3 4 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Sent false 1001 Acked Write 1 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 1 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Read 1 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Write 1 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1002)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:15.207339776Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:15.207487608Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1245,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 45 45 45 F F F 1 020-039 26 26 26 45 45 45 T T F <--- 2 040-059 18 18 18 30 30 30 F F F 3 060-079 17 17 17 27 27 27 T T F <--- 4 080-099 27 27 27 46 46 46 T T F <--- 5 100-119 24 24 24 40 40 40 F F F 6 120-139 24 24 24 40 40 40 F F F 7 140-159 19 19 19 31 31 31 F F F 8 160-179 19 19 19 31 31 31 T T F <--- 9 180-199 25 25 25 41 41 41 T T F <--- 10 200-219 26 26 26 42 42 42 F F F 11 220-239 26 26 26 42 42 42 T T F <--- 12 240-259 22 22 22 36 36 36 T T F <--- 13 260-279 27 27 27 46 46 46 F F F 14 280-299 18 18 18 30 30 30 T T F <--- 15 300-319 26 26 26 42 42 42 F F F 16 320-339 25 25 25 41 41 41 F F F 17 340-359 26 26 26 42 42 42 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 28 28 27 47 47 46 F F F <--- 20 400-419 28 28 15 47 47 25 F F F <--- 21 420-439 28 28 20 47 47 32 F F F <--- 22 440-459 22 22 22 37 37 37 F F F 23 460-479 27 27 27 46 46 46 F F F 24 480-499 24 24 24 39 39 39 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 22 22 22 35 35 35 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 23 23 23 38 38 38 F F F 29 580-599 24 24 24 39 39 39 T T F <--- Max gen: 28, Max flush: 47 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 45 45 45 F F F 1 020-039 29 29 29 48 48 48 F F F 2 040-059 18 18 18 30 30 30 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 29 29 29 48 48 48 F F F 5 100-119 24 24 24 40 40 40 F F F 6 120-139 24 24 24 40 40 40 F F F 7 140-159 19 19 19 31 31 31 F F F 8 160-179 29 29 29 48 48 48 F F F 9 180-199 29 29 29 48 48 48 F F F 10 200-219 26 26 26 42 42 42 F F F 11 220-239 29 29 29 48 48 48 F F F 12 240-259 29 29 29 48 48 48 F F F 13 260-279 27 27 27 46 46 46 F F F 14 280-299 29 29 29 48 48 48 F F F 15 300-319 26 26 26 42 42 42 F F F 16 320-339 25 25 25 41 41 41 F F F 17 340-359 26 26 26 42 42 42 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 28 28 28 47 47 47 F F F 20 400-419 28 28 28 47 47 47 F F F 21 420-439 28 28 28 47 47 47 F F F 22 440-459 22 22 22 37 37 37 F F F 23 460-479 27 27 27 46 46 46 F F F 24 480-499 24 24 24 39 39 39 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 22 22 22 35 35 35 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 23 23 23 38 38 38 F F F 29 580-599 29 29 29 48 48 48 F F F Max gen: 29, Max flush: 48 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.451241309Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.459399607Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.459685284Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.45971406Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.459722799Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.459736578Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253} {"msg":"Crucible 7bfaf021-7f71-4076-9c84-d55702377a2c has session id: 5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460042652Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"Upstairs opts: Upstairs UUID: 7bfaf021-7f71-4076-9c84-d55702377a2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460071699Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"Crucible stats registered with UUID: 7bfaf021-7f71-4076-9c84-d55702377a2c","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460082807Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:19.460092076Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460214342Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.46022916Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460239399Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460419648Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460435246Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253} {"msg":"7bfaf021-7f71-4076-9c84-d55702377a2c active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460458823Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460468142Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"client":"0","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460477831Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"client":"1","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.46048738Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"client":"2","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460496999Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460506938Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460518736Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:58856)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460584209Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:56052)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460609406Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"ds_connection connected from Ok(127.0.0.1:37553)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.460671439Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461280808Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"client":"0","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461306065Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"client":"2","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461317933Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"client":"1","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461592011Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.46160173Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"[0]R flush_numbers[0..12]: [45, 48, 30, 48, 48, 40, 40, 31, 48, 48, 42, 48]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461652104Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"downstairs","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"[0]R generation[0..12]: [26, 29, 18, 29, 29, 24, 24, 19, 29, 29, 26, 29]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461663483Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"downstairs","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461674622Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"downstairs","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"[1]R flush_numbers[0..12]: [45, 48, 30, 48, 48, 40, 40, 31, 48, 48, 42, 48]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.46168499Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"downstairs","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"[1]R generation[0..12]: [26, 29, 18, 29, 29, 24, 24, 19, 29, 29, 26, 29]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461695289Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"downstairs","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461708538Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"downstairs","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"[2]R flush_numbers[0..12]: [45, 48, 30, 48, 48, 40, 40, 31, 48, 48, 42, 48]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461719156Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"downstairs","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"[2]R generation[0..12]: [26, 29, 18, 29, 29, 24, 24, 19, 29, 29, 26, 29]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461729475Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"downstairs","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461740264Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"downstairs","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461749623Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"downstairs","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461759172Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"downstairs","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461768621Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"downstairs","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461778059Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"":"downstairs","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461788088Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461796947Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"client":"0","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461806786Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"client":"1","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461816625Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"client":"2","":"downstairs","session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461826644Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461835873Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"7bfaf021-7f71-4076-9c84-d55702377a2c is now active with session: 5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa","v":0,"name":"crucible","level":30,Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "time":"2026-09-16T04:40:19.461845802Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.46186174Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"session_id":"5f879903-6b5c-48e4-ad9c-cdadf4c7f5aa"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461871089Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253} {"msg":"Activated sub_volume 7bfaf021-7f71-4076-9c84-d55702377a2c","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461879288Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.461887427Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.46211638Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"task":"crutest"} 01/30 Write block 268 len 20480 data: 4 5 6 5 5 02/30 Write block 570 len 32768 data: 3 4 5 5 5 6 4 4 03/30 Write block 118 len 8192 data: 4 3 04/30 Write block 239 len 4096 data: 4 05/30 Flush 06/30 Write block 202 len 24576 data: 4 4 4 3 3 3 07/30 Write block 275 len 24576 data: 6 6 5 5 5 4 08/30 Read block 559 len 36864 09/30 Write block 123 len 12288 data: 4 4 4 10/30 Read block 574 len 12288 11/30 Read block 440 len 16384 12/30 Write block 589 len 16384 data: 5 5 4 5 13/30 Read block 340 len 40960 14/30 Read block 120 len 28672 15/30 Write block 299 len 20480 data: 5 5 5 5 5 16/30 Read block 182 len 28672 17/30 Read block 321 len 4096 18/30 Flush 19/30 Write block 522 len 16384 data: 3 3 3 3 20/30 Write block 189 len 36864 data: 6 5 5 5 5 4 4 3 3 21/30 Flush 22/30 Write block 349 len 40960 data: 3 3 2 2 2 4 2 2 2 4 23/30 Read block 160 len 24576 24/30 Write block 172 len 40960 data: 2 2 2 2 3 3 3 2 2 3 25/30 Read block 89 len 36864 26/30 Read block 26 len 4096 27/30 Write block 13 len 32768 data: 5 5 3 3 3 3 3 3 28/30 Read block 497 len 12288 29/30 Read block 351 len 16384 30/30 Read block 509 len 40960 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 8 Done Sent Done false 1002 Acked Write 2 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Read 3 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Read 1 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 9 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 1 30 Done 30 1 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:19.471643878Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:19.471847704Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1253,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 26 45 45 45 T F T <--- 1 020-039 29 29 29 48 48 48 T F T <--- 2 040-059 18 18 18 30 30 30 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 29 29 29 48 48 48 F F F 5 100-119 30 24 30 49 40 49 F F F <--- 6 120-139 30 24 30 50 40 50 F F F <--- 7 140-159 19 19 19 31 31 31 F F F 8 160-179 29 29 29 48 48 48 T F T <--- 9 180-199 30 29 30 51 48 51 T F T <--- 10 200-219 30 26 30 50 42 50 F F F <--- 11 220-239 30 29 30 49 48 49 F F F <--- 12 240-259 29 29 29 48 48 48 F F F 13 260-279 30 27 30 50 46 50 F T F <--- 14 280-299 30 29 30 50 48 50 F F F <--- 15 300-319 30 26 30 50 42 50 F F F <--- 16 320-339 25 25 25 41 41 41 F F F 17 340-359 26 26 26 42 42 42 T F T <--- 18 360-379 17 17 17 27 27 27 F F F 19 380-399 28 28 28 47 47 47 F F F 20 400-419 28 28 28 47 47 47 F F F 21 420-439 28 28 28 47 47 47 F F F 22 440-459 22 22 22 37 37 37 F F F 23 460-479 27 27 27 46 46 46 F F F 24 480-499 24 24 24 39 39 39 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 30 22 30 51 35 51 F F F <--- 27 540-559 19 19 19 31 31 31 F F F 28 560-579 30 23 30 49 38 49 F F F <--- 29 580-599 30 29 30 50 48 50 F F F <--- Max gen: 30, Max flush: 51 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 52 52 52 F F F 1 020-039 31 31 31 52 52 52 F F F 2 040-059 18 18 18 30 30 30 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 29 29 29 48 48 48 F F F 5 100-119 30 30 30 49 49 49 F F F 6 120-139 30 30 30 50 50 50 F F F 7 140-159 19 19 19 31 31 31 F F F 8 160-179 31 31 31 52 52 52 F F F 9 180-199 31 31 31 52 52 52 F F F 10 200-219 30 30 30 50 50 50 F F F 11 220-239 30 30 30 49 49 49 F F F 12 240-259 29 29 29 48 48 48 F F F 13 260-279 30 30 30 50 50 50 F F F 14 280-299 30 30 30 50 50 50 F F F 15 300-319 30 30 30 50 50 50 F F F 16 320-339 25 25 25 41 41 41 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 28 28 28 47 47 47 F F F 20 400-419 28 28 28 47 47 47 F F F 21 420-439 28 28 28 47 47 47 F F F 22 440-459 22 22 22 37 37 37 F F F 23 460-479 27 27 27 46 46 46 F F F 24 480-499 24 24 24 39 39 39 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 30 30 30 49 49 49 F F F 29 580-599 30 30 30 50 50 50 F F F Max gen: 31, Max flush: 52 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.720983307Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.729171711Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.729450669Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.729477215Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.729485934Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.729497483Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261} {"msg":"Crucible 16603043-c0fc-4863-9b3b-9e2ba65c2638 has session id: 8c25e889-e23e-431c-b801-334f9899db8d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.729816756Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"Upstairs opts: Upstairs UUID: 16603043-c0fc-4863-9b3b-9e2ba65c2638, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.729846662Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"Crucible stats registered with UUID: 16603043-c0fc-4863-9b3b-9e2ba65c2638","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.729858001Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:23.72986738Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730001174Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730031731Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730042599Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730152277Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730167235Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261} {"msg":"16603043-c0fc-4863-9b3b-9e2ba65c2638 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730184183Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730192952Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"client":"0","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730202591Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"client":"1","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.73021206Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"client":"2","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730221958Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730231907Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730241676Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54361)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730317467Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57707)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730507915Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:39306)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.730530632Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731082718Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"client":"0","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731102226Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"client":"1","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731231381Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"client":"2","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.73140937Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731419389Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"[0]R flush_numbers[0..12]: [52, 52, 30, 48, 48, 49, 50, 31, 52, 52, 50, 49]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731520897Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"downstairs","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"[0]R generation[0..12]: [31, 31, 18, 29, 29, 30, 30, 19, 31, 31, 30, 30]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731532106Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"downstairs","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731543114Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"downstairs","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"[1]R flush_numbers[0..12]: [52, 52, 30, 48, 48, 49, 50, 31, 52, 52, 50, 49]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731553573Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"downstairs","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"[1]R generation[0..12]: [31, 31, 18, 29, 29, 30, 30, 19, 31, 31, 30, 30]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731563792Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"downstairs","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731574741Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"downstairs","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"[2]R flush_numbers[0..12]: [52, 52, 30, 48, 48, 49, 50, 31, 52, 52, 50, 49]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731585179Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"downstairs","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"[2]R generation[0..12]: [31, 31, 18, 29, 29, 30, 30, 19, 31, 31, 30, 30]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731595438Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"downstairs","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731606347Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"downstairs","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731615756Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"downstairs","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731626605Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"downstairs","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731635893Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"downstairs","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731645182Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"":"downstairs","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731655231Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.7316641Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"client":"0","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731674049Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"client":"1","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731683618Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"client":"2","":"downstairs","session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731693487Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731702566Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"16603043-c0fc-4863-9b3b-9e2ba65c2638 is now active with session: 8c25e889-e23e-431c-b801-334f9899db8d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731712515Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731721253Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"session_id":"8c25e889-e23e-431c-b801-334f9899db8d"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731730352Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261} {"msg":"Activated sub_volume 16603043-c0fc-4863-9b3b-9e2ba65c2638","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731738232Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.731746431Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.732054385Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"task":"crutest"} 01/30 Write block 83 len 40960 data: 6 5 5 6 7 7 8 6 4 5 02/30 Read block 497 len 20480 03/30 Read block 81 len 20480 04/30 Flush 05/30 Write block 123 len 28672 data: 5 5 5 5 4 4 4 06/30 Write block 158 len 28672 data: 4 5 5 5 4 4 4 07/30 Write block 38 len 36864 data: 2 2 3 3 3 3 5 6 6 08/30 Read block 214 len 28672 09/30 Read block 317 len 24576 10/30 Write block 418 len 16384 data: 2 2 2 2 11/30 Read block 368 len 20480 12/30 Write block 571 len 28672 data: 5 6 6 6 7 5 5 13/30 Read block 162 len 4096 14/30 Read block 520 len 36864 15/30 Read block 56 len 12288 16/30 Flush 17/30 Flush 18/30 Write block 168 len 8192 data: 3 2 19/30 Write block 579 len 32768 data: 5 4 4 4 5 4 5 4 20/30 Read block 421 len 36864 21/30 Read block 27 len 40960 22/30 Read block 79 len 20480 23/30 Write block 450 len 8192 data: 4 3 24/30 Read block 367 len 32768 25/30 Read block 99 len 32768 26/30 Read block 352 len 20480 27/30 Write block 587 len 12288 data: 4 4 6 28/30 Read block 556 len 12288 29/30 Read block 425 len 4096 30/30 Write block 224 len 24576 data: 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 5 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 7 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:23.741253561Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:23.741393574Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1261,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 52 52 52 F F F 1 020-039 32 32 31 54 54 52 F F F <--- 2 040-059 32 32 18 54 54 30 F F F <--- 3 060-079 29 29 29 48 48 48 F F F 4 080-099 32 32 29 53 53 48 F F T <--- 5 100-119 30 30 30 49 49 49 F F F 6 120-139 32 32 30 54 54 50 F F F <--- 7 140-159 32 32 19 54 54 31 F F F <--- 8 160-179 32 32 31 54 54 52 T T F <--- 9 180-199 31 31 31 52 52 52 F F F 10 200-219 30 30 30 50 50 50 F F F 11 220-239 30 30 30 49 49 49 T T F <--- 12 240-259 29 29 29 48 48 48 F F F 13 260-279 30 30 30 50 50 50 F F F 14 280-299 30 30 30 50 50 50 F F F 15 300-319 30 30 30 50 50 50 F F F 16 320-339 25 25 25 41 41 41 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 28 28 28 47 47 47 F F F 20 400-419 32 32 28 54 54 47 F F F <--- 21 420-439 32 32 28 54 54 47 F F F <--- 22 440-459 22 22 22 37 37 37 T T F <--- 23 460-479 27 27 27 46 46 46 F F F 24 480-499 24 24 24 39 39 39 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 32 32 30 54 54 49 T T F <--- 29 580-599 30 30 30 50 50 50 T T F <--- Max gen: 32, Max flush: 54 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 52 52 52 F F F 1 020-039 32 32 32 54 54 54 F F F 2 040-059 32 32 32 54 54 54 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 32 32 32 53 53 53 F F F 5 100-119 30 30 30 49 49 49 F F F 6 120-139 32 32 32 54 54 54 F F F 7 140-159 32 32 32 54 54 54 F F F 8 160-179 33 33 33 55 55 55 F F F 9 180-199 31 31 31 52 52 52 F F F 10 200-219 30 30 30 50 50 50 F F F 11 220-239 33 33 33 55 55 55 F F F 12 240-259 29 29 29 48 48 48 F F F 13 260-279 30 30 30 50 50 50 F F F 14 280-299 30 30 30 50 50 50 F F F 15 300-319 30 30 30 50 50 50 F F F 16 320-339 25 25 25 41 41 41 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 28 28 28 47 47 47 F F F 20 400-419 32 32 32 54 54 54 F F F 21 420-439 32 32 32 54 54 54 F F F 22 440-459 33 33 33 55 55 55 F F F 23 460-479 27 27 27 46 46 46 F F F 24 480-499 24 24 24 39 39 39 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 33 33 33 55 55 55 F F F 29 580-599 33 33 33 55 55 55 F F F Max gen: 33, Max flush: 55 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.981805675Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.989876303Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990162529Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990186447Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990197595Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990209254Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269} {"msg":"Crucible 9b8e259b-d8a1-4a66-8141-f413feda681f has session id: 01299aa2-98d2-456a-8e49-a0d8ff2f6e56","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990555484Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"Upstairs opts: Upstairs UUID: 9b8e259b-d8a1-4a66-8141-f413feda681f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990578321Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"Crucible stats registered with UUID: 9b8e259b-d8a1-4a66-8141-f413feda681f","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.99058833Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:27.990597609Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990737832Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.99075347Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990763489Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990908662Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.99092513Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269} {"msg":"9b8e259b-d8a1-4a66-8141-f413feda681f active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990956927Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990966416Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"client":"0","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990976065Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"client":"1","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990987413Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"client":"2","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.990997012Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.991007181Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.99101702Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"ds_connection connected from Ok(127.0.0.1:37236)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.99110234Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"ds_connection connected from Ok(127.0.0.1:36612)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.991128127Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"ds_connection connected from Ok(127.0.0.1:51041)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.991139625Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.991729517Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"client":"0","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.991755174Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"client":"1","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.991879419Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"client":"2","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992049779Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992080596Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"[0]R flush_numbers[0..12]: [52, 54, 54, 48, 53, 49, 54, 54, 55, 52, 50, 55]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992156177Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"downstairs","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"[0]R generation[0..12]: [31, 32, 32, 29, 32, 30, 32, 32, 33, 31, 30, 33]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992168085Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"downstairs","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992181854Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"downstairs","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"[1]R flush_numbers[0..12]: [52, 54, 54, 48, 53, 49, 54, 54, 55, 52, 50, 55]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992192483Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"downstairs","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"[1]R generation[0..12]: [31, 32, 32, 29, 32, 30, 32, 32, 33, 31, 30, 33]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992202891Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"downstairs","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.99221369Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"downstairs","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"[2]R flush_numbers[0..12]: [52, 54, 54, 48, 53, 49, 54, 54, 55, 52, 50, 55]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992224319Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"downstairs","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"[2]R generation[0..12]: [31, 32, 32, 29, 32, 30, 32, 32, 33, 31, 30, 33]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992234818Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"downstairs","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992245656Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"downstairs","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992255065Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"downstairs","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992264764Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"downstairs","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992274203Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"downstairs","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992283542Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"":"downstairs","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992293391Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.99230219Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"client":"0","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992311929Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"client":"1","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992321777Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"client":"2","":"downstairs","session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992332706Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992341935Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"9b8e259b-d8a1-4a66-8141-f413feda681f is now active with session: 01299aa2-98d2-456a-8e49-a0d8ff2f6e56","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992351784Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992360473Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"session_id":"01299aa2-98d2-456a-8e49-a0d8ff2f6e56"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992369562Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269} {"msg":"Activated sub_volume 9b8e259b-d8a1-4a66-8141-f413feda681f","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992377591Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"2026-09-16T04:40:27.99238579Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:27.992631081Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"task":"crutest"} 01/30 Write block 288 len 36864 data: 3 3 2 2 2 3 3 4 4 02/30 Write block 446 len 12288 data: 4 4 5 03/30 Read block 94 len 36864 04/30 Read block 338 len 28672 05/30 Read block 171 len 12288 06/30 Flush 07/30 Flush 08/30 Write block 50 len 12288 data: 3 5 4 09/30 Flush 10/30 Write block 275 len 32768 data: 7 7 6 6 6 5 4 4 11/30 Read block 401 len 20480 12/30 Read block 213 len 8192 13/30 Read block 290 len 36864 14/30 Read block 218 len 20480 15/30 Flush 16/30 Write block 490 len 24576 data: 4 3 6 6 6 6 17/30 Write block 196 len 24576 data: 4 4 5 3 3 3 18/30 Flush 19/30 Write block 238 len 36864 data: 4 5 3 3 4 4 5 4 5 20/30 Read block 205 len 32768 21/30 Read block 30 len 8192 22/30 Flush 23/30 Flush 24/30 Write block 237 len 36864 data: 4 5 6 4 4 5 5 6 5 25/30 Read block 127 len 4096 26/30 Flush 27/30 Write block 436 len 12288 data: 3 2 2 28/30 Write block 305 len 24576 data: 4 3 3 3 3 2 29/30 Write block 580 len 16384 data: 5 5 5 6 30/30 Write block 272 len 40960 data: 6 6 6 8 8 7 7 7 6 5 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 9 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 3 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 9 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Read 8 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 28 34 Done 27 27 2 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:28.002001967Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:28.002157749Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1269,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 52 52 52 F F F 1 020-039 32 32 32 54 54 54 F F F 2 040-059 34 34 32 58 58 54 F F F <--- 3 060-079 29 29 29 48 48 48 F F F 4 080-099 32 32 32 53 53 53 F F F 5 100-119 30 30 30 49 49 49 F F F 6 120-139 32 32 32 54 54 54 F F F 7 140-159 32 32 32 54 54 54 F F F 8 160-179 33 33 33 55 55 55 F F F 9 180-199 34 34 31 60 60 52 F F F <--- 10 200-219 34 34 30 60 60 50 F F F <--- 11 220-239 34 34 33 63 63 55 F F F <--- 12 240-259 34 34 29 63 63 48 F F F <--- 13 260-279 34 34 30 59 59 50 T T F <--- 14 280-299 34 34 30 59 59 50 T T T <--- 15 300-319 30 30 30 50 50 50 T T F <--- 16 320-339 25 25 25 41 41 41 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 28 28 28 47 47 47 F F F 20 400-419 32 32 32 54 54 54 F F F 21 420-439 32 32 32 54 54 54 T T F <--- 22 440-459 34 34 33 56 56 55 F F T <--- 23 460-479 27 27 27 46 46 46 F F F 24 480-499 34 34 24 60 60 39 F F F <--- 25 500-519 20 20 20 33 33 33 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 33 33 33 55 55 55 F F F 29 580-599 33 33 33 55 55 55 T T F <--- Max gen: 34, Max flush: 63 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 52 52 52 F F F 1 020-039 32 32 32 54 54 54 F F F 2 040-059 34 34 34 58 58 58 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 32 32 32 53 53 53 F F F 5 100-119 30 30 30 49 49 49 F F F 6 120-139 32 32 32 54 54 54 F F F 7 140-159 32 32 32 54 54 54 F F F 8 160-179 33 33 33 55 55 55 F F F 9 180-199 34 34 34 60 60 60 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 34 34 34 63 63 63 F F F 12 240-259 34 34 34 63 63 63 F F F 13 260-279 35 35 35 64 64 64 F F F 14 280-299 35 35 35 64 64 64 F F F 15 300-319 35 35 35 64 64 64 F F F 16 320-339 25 25 25 41 41 41 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 28 28 28 47 47 47 F F F 20 400-419 32 32 32 54 54 54 F F F 21 420-439 35 35 35 64 64 64 F F F 22 440-459 34 34 34 56 56 56 F F F 23 460-479 27 27 27 46 46 46 F F F 24 480-499 34 34 34 60 60 60 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 33 33 33 55 55 55 F F F 29 580-599 35 35 35 64 64 64 F F F Max gen: 35, Max flush: 64 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.24959636Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:32.256744876Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.256786801Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.263722981Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.263971862Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264007218Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264015677Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264027146Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277} {"msg":"Crucible 196fd6ed-b1f7-49ae-b189-121bf1f7da9c has session id: 110c11c7-e6e2-4442-b900-248becd021f3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.26433613Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"Upstairs opts: Upstairs UUID: 196fd6ed-b1f7-49ae-b189-121bf1f7da9c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264361587Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"Crucible stats registered with UUID: 196fd6ed-b1f7-49ae-b189-121bf1f7da9c","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264371725Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:32.264380734Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264514919Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264529427Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264539486Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264691198Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264709956Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277} {"msg":"196fd6ed-b1f7-49ae-b189-121bf1f7da9c active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264733443Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264742792Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"client":"0","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264752511Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"client":"1","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.26476203Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"client":"2","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264771519Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264781398Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264791266Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:64029)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264858199Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43295)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.264870747Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:64548)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.265005571Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.265541479Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"client":"2","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.265592953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"client":"1","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.265604931Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"client":"0","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.265846633Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.26587786Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"[0]R flush_numbers[0..12]: [52, 54, 58, 48, 53, 49, 54, 54, 55, 60, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.265926894Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"downstairs","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"[0]R generation[0..12]: [31, 32, 34, 29, 32, 30, 32, 32, 33, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.265938233Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"downstairs","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.265949541Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"downstairs","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"[1]R flush_numbers[0..12]: [52, 54, 58, 48, 53, 49, 54, 54, 55, 60, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.26595992Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"downstairs","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"[1]R generation[0..12]: [31, 32, 34, 29, 32, 30, 32, 32, 33, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.265970369Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"downstairs","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.265981238Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"downstairs","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"[2]R flush_numbers[0..12]: [52, 54, 58, 48, 53, 49, 54, 54, 55, 60, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.265991576Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"downstairs","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"[2]R generation[0..12]: [31, 32, 34, 29, 32, 30, 32, 32, 33, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266004655Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"downstairs","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266015714Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"downstairs","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266025112Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"downstairs","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266034961Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"downstairs","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"Next flush: 65","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.26604425Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"downstairs","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266053639Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"":"downstairs","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266063328Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266072137Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"client":"0","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266081706Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"client":"1","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266091285Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"client":"2","":"downstairs","session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266101174Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266110242Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"196fd6ed-b1f7-49ae-b189-121bf1f7da9c is now active with session: 110c11c7-e6e2-4442-b900-248becd021f3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266120061Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.26612884Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"session_id":"110c11c7-e6e2-4442-b900-248becd021f3"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266138069Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277} {"msg":"Activated sub_volume 196fd6ed-b1f7-49ae-b189-121bf1f7da9c","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266146168Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266155887Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.266464061Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"task":"crutest"} 01/30 Write block 317 len 12288 data: 4 4 5 02/30 Read block 86 len 40960 03/30 Read block 214 len 4096 04/30 Write block 248 len 20480 data: 4 4 5 6 6 05/30 Write block 287 len 20480 data: 4 4 4 3 3 06/30 Read block 57 len 20480 07/30 Read block 535 len 16384 08/30 Read block 94 len 28672 09/30 Read block 64 len 36864 10/30 Read block 145 len 16384 11/30 Read block 420 len 28672 12/30 Read block 404 len 8192 13/30 Read block 574 len 36864 14/30 Write block 165 len 36864 data: 4 4 4 4 3 2 2 3 3 15/30 Read block 177 len 4096 16/30 Write block 275 len 24576 data: 9 9 8 8 8 7 17/30 Read block 242 len 24576 18/30 Write block 391 len 4096 data: 4 19/30 Read block 525 len 20480 20/30 Read block 391 len 20480 21/30 Read block 92 len 20480 22/30 Read block 82 len 16384 23/30 Write block 104 len 12288 data: 4 3 3 24/30 Write block 285 len 40960 data: 3 4 5 5 5 4 4 3 4 4 25/30 Read block 522 len 16384 26/30 Write block 472 len 40960 data: 3 4 4 4 6 6 6 6 4 4 27/30 Read block 3 len 12288 28/30 Write block 466 len 16384 data: 3 3 3 3 29/30 Write block 168 len 32768 data: 5 4 3 3 4 4 3 3 30/30 Read block 187 len 20480 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 10 Sent Done Done false 1002 Acked Read 1 Sent Done Done false 1003 Acked Write 5 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 1 30 Done 1 30 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:32.275776044Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:32.275945344Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1277,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 52 52 52 F F F 1 020-039 32 32 32 54 54 54 F F F 2 040-059 34 34 34 58 58 58 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 32 32 32 53 53 53 F F F 5 100-119 30 30 30 49 49 49 F T T <--- 6 120-139 32 32 32 54 54 54 F F F 7 140-159 32 32 32 54 54 54 F F F 8 160-179 33 33 33 55 55 55 F T T <--- 9 180-199 34 34 34 60 60 60 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 34 34 34 63 63 63 F F F 12 240-259 34 34 34 63 63 63 F T T <--- 13 260-279 35 35 35 64 64 64 F T T <--- 14 280-299 35 35 35 64 64 64 F T T <--- 15 300-319 35 35 35 64 64 64 T T T 16 320-339 25 25 25 41 41 41 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 28 28 28 47 47 47 F T T <--- 20 400-419 32 32 32 54 54 54 F F F 21 420-439 35 35 35 64 64 64 F F F 22 440-459 34 34 34 56 56 56 F F F 23 460-479 27 27 27 46 46 46 F T T <--- 24 480-499 34 34 34 60 60 60 F T T <--- 25 500-519 20 20 20 33 33 33 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 33 33 33 55 55 55 F F F 29 580-599 35 35 35 64 64 64 F F F Max gen: 35, Max flush: 64 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 52 52 52 F F F 1 020-039 32 32 32 54 54 54 F F F 2 040-059 34 34 34 58 58 58 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 32 32 32 53 53 53 F F F 5 100-119 36 36 36 65 65 65 F F F 6 120-139 32 32 32 54 54 54 F F F 7 140-159 32 32 32 54 54 54 F F F 8 160-179 36 36 36 65 65 65 F F F 9 180-199 34 34 34 60 60 60 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 34 34 34 63 63 63 F F F 12 240-259 36 36 36 65 65 65 F F F 13 260-279 36 36 36 65 65 65 F F F 14 280-299 36 36 36 65 65 65 F F F 15 300-319 36 36 36 65 65 65 F F F 16 320-339 25 25 25 41 41 41 F F F 17 340-359 31 31 31 52 52 52 F F F 18 360-379 17 17 17 27 27 27 F F F 19 380-399 36 36 36 65 65 65 F F F 20 400-419 32 32 32 54 54 54 F F F 21 420-439 35 35 35 64 64 64 F F F 22 440-459 34 34 34 56 56 56 F F F 23 460-479 36 36 36 65 65 65 F F F 24 480-499 36 36 36 65 65 65 F F F 25 500-519 20 20 20 33 33 33 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 19 19 19 31 31 31 F F F 28 560-579 33 33 33 55 55 55 F F F 29 580-599 35 35 35 64 64 64 F F F Max gen: 36, Max flush: 65 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.523422151Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:36.530437452Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.530485256Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.537314109Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.5375648Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.537594147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.537602926Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.537614364Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285} {"msg":"Crucible 642f4331-4e47-426c-9035-1a08f6cb7e2d has session id: e440f595-ab80-45a8-8e1c-a43bc5f5c75d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.537951235Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"Upstairs opts: Upstairs UUID: 642f4331-4e47-426c-9035-1a08f6cb7e2d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.537976072Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"Crucible stats registered with UUID: 642f4331-4e47-426c-9035-1a08f6cb7e2d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.537986111Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:36.53799519Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538145612Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538175779Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538186667Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538303264Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538318602Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285} {"msg":"642f4331-4e47-426c-9035-1a08f6cb7e2d active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538356358Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538365887Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"client":"0","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538375585Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"client":"1","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538385234Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"client":"2","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538394783Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.53841973Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538460046Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45866)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538470844Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"ds_connection connected from Ok(127.0.0.1:40565)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538484773Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56267)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.538495111Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539160944Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"client":"1","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539180811Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"client":"0","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.53919399Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"client":"2","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.53945367Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539478217Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"[0]R flush_numbers[0..12]: [52, 54, 58, 48, 53, 65, 54, 54, 65, 60, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539530801Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"downstairs","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"[0]R generation[0..12]: [31, 32, 34, 29, 32, 36, 32, 32, 36, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539541959Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"downstairs","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539552838Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"downstairs","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"[1]R flush_numbers[0..12]: [52, 54, 58, 48, 53, 65, 54, 54, 65, 60, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539563297Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"downstairs","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"[1]R generation[0..12]: [31, 32, 34, 29, 32, 36, 32, 32, 36, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539573726Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"downstairs","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539584604Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"downstairs","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"[2]R flush_numbers[0..12]: [52, 54, 58, 48, 53, 65, 54, 54, 65, 60, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539595043Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"downstairs","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"[2]R generation[0..12]: [31, 32, 34, 29, 32, 36, 32, 32, 36, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539605682Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"downstairs","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539616641Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"downstairs","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"Max found gen is 37","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539626169Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"downstairs","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"Generation requested: 38 >= found:37","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539635918Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"downstairs","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539646477Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"downstairs","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539656096Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"":"downstairs","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539665825Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539674594Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"client":"0","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539684213Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"client":"1","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539693891Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"client":"2","":"downstairs","session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.53970376Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539712929Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"642f4331-4e47-426c-9035-1a08f6cb7e2d is now active with session: e440f595-ab80-45a8-8e1c-a43bc5f5c75d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539722858Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539731667Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"session_id":"e440f595-ab80-45a8-8e1c-a43bc5f5c75d"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539740936Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285} {"msg":"Activated sub_volume 642f4331-4e47-426c-9035-1a08f6cb7e2d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539749065Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.539757224Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.54005048Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"task":"crutest"} 01/30 Write block 364 len 32768 data: 3 3 3 3 4 4 3 3 02/30 Read block 104 len 20480 03/30 Read block 560 len 12288 04/30 Write block 567 len 24576 data: 3 3 2 4 6 7 05/30 Read block 505 len 16384 06/30 Write block 159 len 8192 data: 6 6 07/30 Read block 291 len 36864 08/30 Read block 532 len 20480 09/30 Read block 13 len 16384 10/30 Write block 412 len 4096 data: 3 11/30 Read block 498 len 4096 12/30 Write block 145 len 16384 data: 4 4 4 4 13/30 Read block 486 len 32768 14/30 Write block 351 len 24576 data: 3 3 3 5 3 3 15/30 Write block 279 len 4096 data: 9 16/30 Read block 580 len 28672 17/30 Read block 96 len 20480 18/30 Read block 142 len 16384 19/30 Read block 464 len 24576 20/30 Write block 478 len 12288 data: 7 7 5 21/30 Write block 503 len 4096 data: 2 22/30 Write block 328 len 36864 data: 4 3 4 3 3 5 4 4 4 23/30 Flush 24/30 Write block 383 len 20480 data: 5 5 4 4 4 25/30 Write block 29 len 20480 data: 6 6 7 7 7 26/30 Write block 51 len 36864 data: 6 5 4 4 4 5 5 4 3 27/30 Read block 517 len 28672 28/30 Write block 551 len 28672 data: 4 4 5 5 6 5 4 29/30 Read block 24 len 20480 30/30 Write block 352 len 40960 data: 4 4 6 4 4 3 5 4 5 5 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 6 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Write 2 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Write 1 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Write 4 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Write 1 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 5 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 1 29 Done 3 29 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:36.548726297Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:36.548918365Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1285,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 52 52 52 F F F 1 020-039 32 32 32 54 54 54 F T T <--- 2 040-059 34 34 34 58 58 58 F T T <--- 3 060-079 29 29 29 48 48 48 F F F 4 080-099 32 32 32 53 53 53 F F F 5 100-119 36 36 36 65 65 65 F F F 6 120-139 32 32 32 54 54 54 F F F 7 140-159 32 38 38 54 66 66 F F F <--- 8 160-179 36 38 38 65 66 66 F F F <--- 9 180-199 34 34 34 60 60 60 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 34 34 34 63 63 63 F F F 12 240-259 36 36 36 65 65 65 F F F 13 260-279 36 38 38 65 66 66 F F F <--- 14 280-299 36 36 36 65 65 65 F F F 15 300-319 36 36 36 65 65 65 F F F 16 320-339 25 38 38 41 66 66 F F F <--- 17 340-359 31 38 38 52 66 66 F T T <--- 18 360-379 17 38 38 27 66 66 T T T <--- 19 380-399 36 36 36 65 65 65 F T T <--- 20 400-419 32 38 38 54 66 66 F F F <--- 21 420-439 35 35 35 64 64 64 F F F 22 440-459 34 34 34 56 56 56 F F F 23 460-479 36 38 38 65 66 66 F F F <--- 24 480-499 36 38 38 65 66 66 F F F <--- 25 500-519 20 38 38 33 66 66 F F F <--- 26 520-539 30 30 30 51 51 51 F F F 27 540-559 19 19 19 31 31 31 F T T <--- 28 560-579 33 38 38 55 66 66 F F F <--- 29 580-599 35 35 35 64 64 64 F F F Max gen: 38, Max flush: 66 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 31 31 31 52 52 52 F F F 1 020-039 39 39 39 67 67 67 F F F 2 040-059 39 39 39 67 67 67 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 32 32 32 53 53 53 F F F 5 100-119 36 36 36 65 65 65 F F F 6 120-139 32 32 32 54 54 54 F F F 7 140-159 38 38 38 66 66 66 F F F 8 160-179 38 38 38 66 66 66 F F F 9 180-199 34 34 34 60 60 60 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 34 34 34 63 63 63 F F F 12 240-259 36 36 36 65 65 65 F F F 13 260-279 38 38 38 66 66 66 F F F 14 280-299 36 36 36 65 65 65 F F F 15 300-319 36 36 36 65 65 65 F F F 16 320-339 38 38 38 66 66 66 F F F 17 340-359 39 39 39 67 67 67 F F F 18 360-379 39 39 39 67 67 67 F F F 19 380-399 39 39 39 67 67 67 F F F 20 400-419 38 38 38 66 66 66 F F F 21 420-439 35 35 35 64 64 64 F F F 22 440-459 34 34 34 56 56 56 F F F 23 460-479 38 38 38 66 66 66 F F F 24 480-499 38 38 38 66 66 66 F F F 25 500-519 38 38 38 66 66 66 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 39 39 39 67 67 67 F F F 28 560-579 38 38 38 66 66 66 F F F 29 580-599 35 35 35 64 64 64 F F F Max gen: 39, Max flush: 67 Begin loop 20 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.805792114Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.814041621Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.814324098Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.814348296Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.814358944Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.814370433Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293} {"msg":"Crucible c947c749-b5a2-44b7-8cc3-6a8a025c0927 has session id: f9114df4-e376-4541-af04-006bf16b8c88","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.814711633Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"Upstairs opts: Upstairs UUID: c947c749-b5a2-44b7-8cc3-6a8a025c0927, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.81474052Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"Crucible stats registered with UUID: c947c749-b5a2-44b7-8cc3-6a8a025c0927","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.814751648Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:40.814760827Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.814887973Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.814916839Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.814927728Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.815066182Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.81508042Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293} {"msg":"c947c749-b5a2-44b7-8cc3-6a8a025c0927 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.815114026Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.815123595Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"client":"0","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.815133244Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"client":"1","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.815145392Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"client":"2","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.815155041Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.81516502Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.815175069Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"ds_connection connected from Ok(127.0.0.1:62347)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.815231892Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"ds_connection connected from Ok(127.0.0.1:61691)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.815257999Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"ds_connection connected from Ok(127.0.0.1:64825)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.815269388Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.815886736Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"client":"2","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.815908593Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"client":"0","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.815944979Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"client":"1","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.81619763Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816222907Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"[0]R flush_numbers[0..12]: [52, 67, 67, 48, 53, 65, 54, 66, 66, 60, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816259512Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"downstairs","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"[0]R generation[0..12]: [31, 39, 39, 29, 32, 36, 32, 38, 38, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816270621Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"downstairs","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.81628428Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"downstairs","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"[1]R flush_numbers[0..12]: [52, 67, 67, 48, 53, 65, 54, 66, 66, 60, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816294818Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"downstairs","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"[1]R generation[0..12]: [31, 39, 39, 29, 32, 36, 32, 38, 38, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816305257Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"downstairs","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816316296Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"downstairs","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"[2]R flush_numbers[0..12]: [52, 67, 67, 48, 53, 65, 54, 66, 66, 60, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816326615Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"downstairs","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"[2]R generation[0..12]: [31, 39, 39, 29, 32, 36, 32, 38, 38, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816336923Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"downstairs","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816347712Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"downstairs","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816357231Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"downstairs","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.81636695Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"downstairs","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816376249Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"downstairs","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816385488Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"":"downstairs","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816395317Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816403996Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"client":"0","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816413564Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"client":"1","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816423183Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"client":"2","":"downstairs","session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816434082Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816443161Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"c947c749-b5a2-44b7-8cc3-6a8a025c0927 is now active with session: f9114df4-e376-4541-af04-006bf16b8c88","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.81645301Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816461679Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"session_id":"f9114df4-e376-4541-af04-006bf16b8c88"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816470618Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293} {"msg":"Activated sub_volume c947c749-b5a2-44b7-8cc3-6a8a025c0927","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816478657Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.816486736Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.81679091Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"task":"crutest"} 01/30 Write block 267 len 36864 data: 6 5 6 7 6 7 7 7 10 02/30 Read block 16 len 24576 03/30 Read block 40 len 12288 04/30 Write block 189 len 16384 data: 7 6 6 6 05/30 Write block 60 len 20480 data: 3 3 3 2 3 06/30 Read block 406 len 8192 07/30 Read block 427 len 24576 08/30 Read block 46 len 40960 09/30 Read block 14 len 24576 10/30 Write block 341 len 4096 data: 3 11/30 Read block 545 len 40960 12/30 Read block 476 len 12288 13/30 Write block 363 len 24576 data: 4 4 4 4 4 5 14/30 Write block 124 len 40960 data: 6 6 6 5 5 5 4 3 3 3 15/30 Read block 449 len 4096 16/30 Write block 305 len 8192 data: 5 4 17/30 Read block 541 len 24576 18/30 Flush 19/30 Write block 1 len 24576 data: 2 3 4 5 4 4 20/30 Read block 128 len 28672 21/30 Write block 483 len 16384 data: 3 3 3 3 22/30 Read block 426 len 8192 23/30 Write block 382 len 8192 data: 5 6 24/30 Flush 25/30 Write block 20 len 8192 data: 4 3 26/30 Read block 265 len 12288 27/30 Write block 441 len 4096 data: 4 28/30 Read block 300 len 36864 29/30 Write block 332 len 8192 data: 4 6 30/30 Read block 298 len 36864 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Sent false 1001 Acked Read 6 Done Done Sent false 1002 Acked Read 3 Done Done Sent false 1003 Acked Write 4 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Read 2 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Write 4 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:40.824939669Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:40.825112499Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1293,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 40 40 31 69 69 52 F F F <--- 1 020-039 39 39 39 67 67 67 T T F <--- 2 040-059 39 39 39 67 67 67 F F F 3 060-079 40 40 29 68 68 48 F F F <--- 4 080-099 32 32 32 53 53 53 F F F 5 100-119 36 36 36 65 65 65 F F F 6 120-139 40 40 32 68 68 54 F F F <--- 7 140-159 38 38 38 66 66 66 F F F 8 160-179 38 38 38 66 66 66 F F F 9 180-199 40 40 34 68 68 60 F F F <--- 10 200-219 34 34 34 60 60 60 F F F 11 220-239 34 34 34 63 63 63 F F F 12 240-259 36 36 36 65 65 65 F F F 13 260-279 40 40 38 68 68 66 F F F <--- 14 280-299 36 36 36 65 65 65 F F F 15 300-319 40 40 36 68 68 65 F F F <--- 16 320-339 38 38 38 66 66 66 T T F <--- 17 340-359 40 40 39 68 68 67 F F F <--- 18 360-379 40 40 39 68 68 67 F F F <--- 19 380-399 40 40 39 69 69 67 F F F <--- 20 400-419 38 38 38 66 66 66 F F F 21 420-439 35 35 35 64 64 64 F F F 22 440-459 34 34 34 56 56 56 T T F <--- 23 460-479 38 38 38 66 66 66 F F F 24 480-499 40 40 38 69 69 66 F F F <--- 25 500-519 38 38 38 66 66 66 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 39 39 39 67 67 67 F F F 28 560-579 38 38 38 66 66 66 F F F 29 580-599 35 35 35 64 64 64 F F F Max gen: 40, Max flush: 69 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 40 40 40 69 69 69 F F F 1 020-039 41 41 41 70 70 70 F F F 2 040-059 39 39 39 67 67 67 F F F 3 060-079 40 40 40 68 68 68 F F F 4 080-099 32 32 32 53 53 53 F F F 5 100-119 36 36 36 65 65 65 F F F 6 120-139 40 40 40 68 68 68 F F F 7 140-159 38 38 38 66 66 66 F F F 8 160-179 38 38 38 66 66 66 F F F 9 180-199 40 40 40 68 68 68 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 34 34 34 63 63 63 F F F 12 240-259 36 36 36 65 65 65 F F F 13 260-279 40 40 40 68 68 68 F F F 14 280-299 36 36 36 65 65 65 F F F 15 300-319 40 40 40 68 68 68 F F F 16 320-339 41 41 41 70 70 70 F F F 17 340-359 40 40 40 68 68 68 F F F 18 360-379 40 40 40 68 68 68 F F F 19 380-399 40 40 40 69 69 69 F F F 20 400-419 38 38 38 66 66 66 F F F 21 420-439 35 35 35 64 64 64 F F F 22 440-459 41 41 41 70 70 70 F F F 23 460-479 38 38 38 66 66 66 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 38 38 38 66 66 66 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 39 39 39 67 67 67 F F F 28 560-579 38 38 38 66 66 66 F F F 29 580-599 35 35 35 64 64 64 F F F Max gen: 41, Max flush: 70 Begin loop 21 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.069250435Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:45.076281534Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.076333138Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.083137114Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.083381015Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.083409782Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.083418161Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.08342975Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301} {"msg":"Crucible e72242fb-5e5b-489e-be1b-dbae031f3ba8 has session id: dc52060a-29df-4e5f-82ee-d1c6e773f92c","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.083723305Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"Upstairs opts: Upstairs UUID: e72242fb-5e5b-489e-be1b-dbae031f3ba8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.083741143Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"Crucible stats registered with UUID: e72242fb-5e5b-489e-be1b-dbae031f3ba8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.083750942Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:45.083760161Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.083906104Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.083935191Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.083946159Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084067765Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084079814Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301} {"msg":"e72242fb-5e5b-489e-be1b-dbae031f3ba8 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084096492Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084105321Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"client":"0","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.08411483Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"client":"1","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084124768Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"client":"2","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084134417Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084144296Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084154365Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37518)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084215028Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62621)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084232586Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52335)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084314916Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084906607Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"client":"1","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084923245Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"client":"2","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.084934064Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"client":"0","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085257786Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085267565Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"[0]R flush_numbers[0..12]: [69, 70, 67, 68, 53, 65, 68, 66, 66, 68, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085279724Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"downstairs","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"[0]R generation[0..12]: [40, 41, 39, 40, 32, 36, 40, 38, 38, 40, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085290242Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"downstairs","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085301201Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"downstairs","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"[1]R flush_numbers[0..12]: [69, 70, 67, 68, 53, 65, 68, 66, 66, 68, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.08531156Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"downstairs","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"[1]R generation[0..12]: [40, 41, 39, 40, 32, 36, 40, 38, 38, 40, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085322309Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"downstairs","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085333237Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"downstairs","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"[2]R flush_numbers[0..12]: [69, 70, 67, 68, 53, 65, 68, 66, 66, 68, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085345916Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"downstairs","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"[2]R generation[0..12]: [40, 41, 39, 40, 32, 36, 40, 38, 38, 40, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085356275Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"downstairs","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085367133Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"downstairs","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085376562Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"downstairs","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085386151Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"downstairs","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.08539534Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"downstairs","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085404899Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"":"downstairs","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085414528Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085423327Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"client":"0","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085432996Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"client":"1","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085442665Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"client":"2","":"downstairs","session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085452543Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085461702Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"e72242fb-5e5b-489e-be1b-dbae031f3ba8 is now active with session: dc52060a-29df-4e5f-82ee-d1c6e773f92c","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085518246Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085527005Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"session_id":"dc52060a-29df-4e5f-82ee-d1c6e773f92c"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085536124Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301} {"msg":"Activated sub_volume e72242fb-5e5b-489e-be1b-dbae031f3ba8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085544063Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-09-16T04:40:45.085553652Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.085811391Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"task":"crutest"} 01/30 Write block 106 len 12288 data: 4 3 3 02/30 Read block 415 len 8192 03/30 Write block 190 len 8192 data: 7 7 04/30 Read block 486 len 12288 05/30 Read block 236 len 20480 06/30 Write block 272 len 12288 data: 8 8 8 07/30 Read block 296 len 16384 08/30 Flush 09/30 Write block 128 len 12288 data: 6 6 5 10/30 Write block 393 len 36864 data: 3 3 3 3 3 5 5 5 5 11/30 Flush 12/30 Write block 112 len 20480 data: 2 2 2 2 3 13/30 Write block 429 len 32768 data: 5 5 5 6 6 4 3 4 14/30 Read block 27 len 12288 15/30 Write block 434 len 16384 data: 5 4 5 3 16/30 Write block 86 len 36864 data: 7 8 8 9 7 5 6 5 5 17/30 Read block 26 len 4096 18/30 Read block 228 len 28672 19/30 Flush 20/30 Write block 159 len 8192 data: 7 7 21/30 Write block 126 len 24576 data: 7 6 7 7 6 4 22/30 Flush 23/30 Write block 443 len 12288 data: 5 4 4 24/30 Read block 333 len 32768 25/30 Flush 26/30 Write block 332 len 36864 data: 5 7 5 5 5 5 4 4 4 27/30 Write block 5 len 36864 data: 5 5 4 4 3 5 5 5 6 28/30 Write block 564 len 12288 data: 5 3 3 29/30 Write block 545 len 36864 data: 5 5 5 4 5 6 5 5 6 30/30 Write block 473 len 8192 data: 5 5 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Write 9 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 2 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Write 9 Sent Sent Sent false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 4 4 35 Done 3 26 26 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:45.094117432Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:45.094292162Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1301,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 40 40 40 69 69 69 F T T <--- 1 020-039 41 41 41 70 70 70 F F F 2 040-059 39 39 39 67 67 67 F F F 3 060-079 40 40 40 68 68 68 F F F 4 080-099 32 42 42 53 73 73 F F F <--- 5 100-119 36 42 42 65 73 73 T F F <--- 6 120-139 40 42 42 68 74 74 F F F <--- 7 140-159 38 42 42 66 74 74 F F F <--- 8 160-179 38 42 42 66 74 74 F F F <--- 9 180-199 40 42 42 68 71 71 T F F <--- 10 200-219 34 34 34 60 60 60 F F F 11 220-239 34 34 34 63 63 63 F F F 12 240-259 36 36 36 65 65 65 F F F 13 260-279 40 42 42 68 71 71 F F F <--- 14 280-299 36 36 36 65 65 65 F F F 15 300-319 40 40 40 68 68 68 F F F 16 320-339 41 41 41 70 70 70 F T T <--- 17 340-359 40 40 40 68 68 68 F T T <--- 18 360-379 40 40 40 68 68 68 F F F 19 380-399 40 42 42 69 72 72 F F F <--- 20 400-419 38 42 42 66 72 72 F F F <--- 21 420-439 35 42 42 64 73 73 F F F <--- 22 440-459 41 42 42 70 75 75 F F F <--- 23 460-479 38 38 38 66 66 66 F T T <--- 24 480-499 40 40 40 69 69 69 F F F 25 500-519 38 38 38 66 66 66 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 39 39 39 67 67 67 F T T <--- 28 560-579 38 38 38 66 66 66 F T T <--- 29 580-599 35 35 35 64 64 64 F F F Max gen: 42, Max flush: 75 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 76 76 76 F F F 1 020-039 41 41 41 70 70 70 F F F 2 040-059 39 39 39 67 67 67 F F F 3 060-079 40 40 40 68 68 68 F F F 4 080-099 42 42 42 73 73 73 F F F 5 100-119 42 42 42 73 73 73 F F F 6 120-139 42 42 42 74 74 74 F F F 7 140-159 42 42 42 74 74 74 F F F 8 160-179 42 42 42 74 74 74 F F F 9 180-199 42 42 42 71 71 71 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 34 34 34 63 63 63 F F F 12 240-259 36 36 36 65 65 65 F F F 13 260-279 42 42 42 71 71 71 F F F 14 280-299 36 36 36 65 65 65 F F F 15 300-319 40 40 40 68 68 68 F F F 16 320-339 43 43 43 76 76 76 F F F 17 340-359 43 43 43 76 76 76 F F F 18 360-379 40 40 40 68 68 68 F F F 19 380-399 42 42 42 72 72 72 F F F 20 400-419 42 42 42 72 72 72 F F F 21 420-439 42 42 42 73 73 73 F F F 22 440-459 42 42 42 75 75 75 F F F 23 460-479 43 43 43 76 76 76 F F F 24 480-499 40 40 40 69 69 69 F F F 25 500-519 38 38 38 66 66 66 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 43 43 43 76 76 76 F F F 28 560-579 43 43 43 76 76 76 F F F 29 580-599 35 35 35 64 64 64 F F F Max gen: 43, Max flush: 76 Begin loop 22 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.359687316Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.367956071Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.368207792Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.368241288Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.368250517Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.368262225Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309} {"msg":"Crucible 4fb134c6-eadb-4b85-97f6-5e3801f418ce has session id: d1b5bc37-9edc-4180-8cf8-958c69db7552","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.368597636Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"Upstairs opts: Upstairs UUID: 4fb134c6-eadb-4b85-97f6-5e3801f418ce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.368616644Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"Crucible stats registered with UUID: 4fb134c6-eadb-4b85-97f6-5e3801f418ce","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.368626693Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:49.368635822Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.368768466Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.368783635Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.368795513Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.368933097Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.368948695Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309} {"msg":"4fb134c6-eadb-4b85-97f6-5e3801f418ce active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.368980762Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.36899022Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"client":"0","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.369000189Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"client":"1","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.369010118Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"client":"2","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.369019657Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.369029466Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.369064242Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"ds_connection connected from Ok(127.0.0.1:35434)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.369087399Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"ds_connection connected from Ok(127.0.0.1:61638)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.369187537Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"ds_connection connected from Ok(127.0.0.1:42495)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.369200716Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.36976675Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"client":"0","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.369816914Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"client":"1","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.369837002Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"client":"2","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370075274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.37010758Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"[0]R flush_numbers[0..12]: [76, 70, 67, 68, 73, 73, 74, 74, 74, 71, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370154315Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"downstairs","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"[0]R generation[0..12]: [43, 41, 39, 40, 42, 42, 42, 42, 42, 42, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370166093Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"downstairs","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370177262Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"downstairs","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"[1]R flush_numbers[0..12]: [76, 70, 67, 68, 73, 73, 74, 74, 74, 71, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370187901Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"downstairs","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"[1]R generation[0..12]: [43, 41, 39, 40, 42, 42, 42, 42, 42, 42, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370198259Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"downstairs","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370209078Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"downstairs","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"[2]R flush_numbers[0..12]: [76, 70, 67, 68, 73, 73, 74, 74, 74, 71, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370219467Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"downstairs","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"[2]R generation[0..12]: [43, 41, 39, 40, 42, 42, 42, 42, 42, 42, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370229746Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"downstairs","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370240615Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"downstairs","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370249933Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"downstairs","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370259622Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"downstairs","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370268961Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"downstairs","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.37028129Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"":"downstairs","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370291259Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370300028Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"client":"0","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370309596Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"client":"1","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370319125Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"client":"2","":"downstairs","session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370328954Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370338173Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"4fb134c6-eadb-4b85-97f6-5e3801f418ce is now active with session: d1b5bc37-9edc-4180-8cf8-958c69db7552","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370348032Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370356761Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"session_id":"d1b5bc37-9edc-4180-8cf8-958c69db7552"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.37036598Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309} {"msg":"Activated sub_volume 4fb134c6-eadb-4b85-97f6-5e3801f418ce","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370373949Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370381948Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.370680553Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"task":"crutest"} 01/30 Flush 02/30 Write block 499 len 8192 data: 3 3 03/30 Flush 04/30 Flush 05/30 Write block 315 len 20480 data: 3 4 5 5 6 06/30 Flush 07/30 Write block 142 len 36864 data: 3 3 4 5 5 5 5 4 6 08/30 Write block 589 len 8192 data: 7 6 09/30 Read block 575 len 40960 10/30 Read block 0 len 20480 11/30 Read block 512 len 28672 12/30 Read block 210 len 16384 13/30 Write block 111 len 36864 data: 2 3 3 3 3 4 4 5 4 14/30 Write block 95 len 24576 data: 6 5 3 4 5 5 15/30 Read block 79 len 32768 16/30 Read block 197 len 36864 17/30 Read block 420 len 40960 18/30 Write block 302 len 28672 data: 6 6 4 6 5 4 4 19/30 Write block 274 len 24576 data: 9 11 10 9 9 10 20/30 Read block 383 len 40960 21/30 Write block 26 len 28672 data: 5 4 5 7 7 8 8 22/30 Write block 163 len 8192 data: 5 5 23/30 Read block 142 len 20480 24/30 Read block 205 len 12288 25/30 Flush 26/30 Write block 458 len 40960 data: 4 5 4 3 4 4 4 3 4 4 27/30 Write block 352 len 12288 data: 5 5 7 28/30 Write block 348 len 36864 data: 3 4 4 4 6 6 8 5 5 29/30 Read block 427 len 24576 30/30 Write block 30 len 16384 data: 8 9 9 8 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 9 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Read 8 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Write 7 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 18 1 20 Done 23 6 23 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1005)) Some(JobId(1024)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:49.380229319Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:49.380406518Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1309,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 76 76 76 F F F 1 020-039 44 41 44 81 70 81 T F T <--- 2 040-059 39 39 39 67 67 67 F F F 3 060-079 40 40 40 68 68 68 F F F 4 080-099 44 42 44 81 73 81 F F F <--- 5 100-119 44 42 44 81 73 81 F F F <--- 6 120-139 42 42 42 74 74 74 F F F 7 140-159 44 42 44 81 74 81 F T F <--- 8 160-179 44 42 44 81 74 81 F F F <--- 9 180-199 42 42 42 71 71 71 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 34 34 34 63 63 63 F F F 12 240-259 36 36 36 65 65 65 F F F 13 260-279 44 42 44 81 71 81 F F F <--- 14 280-299 36 36 36 65 65 65 F F F 15 300-319 44 44 44 81 80 81 F F F <--- 16 320-339 43 43 43 76 76 76 F F F 17 340-359 43 43 43 76 76 76 T F T <--- 18 360-379 40 40 40 68 68 68 F F F 19 380-399 42 42 42 72 72 72 F F F 20 400-419 42 42 42 72 72 72 F F F 21 420-439 42 42 42 73 73 73 F F F 22 440-459 42 42 42 75 75 75 T F T <--- 23 460-479 43 43 43 76 76 76 T F T <--- 24 480-499 44 44 44 78 78 78 F F F 25 500-519 44 44 44 78 78 78 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 43 43 43 76 76 76 F F F 28 560-579 43 43 43 76 76 76 F F F 29 580-599 44 35 44 81 64 81 F T F <--- Max gen: 44, Max flush: 81 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 76 76 76 F F F 1 020-039 45 45 45 82 82 82 F F F 2 040-059 39 39 39 67 67 67 F F F 3 060-079 40 40 40 68 68 68 F F F 4 080-099 44 44 44 81 81 81 F F F 5 100-119 44 44 44 81 81 81 F F F 6 120-139 42 42 42 74 74 74 F F F 7 140-159 44 44 44 81 81 81 F F F 8 160-179 44 44 44 81 81 81 F F F 9 180-199 42 42 42 71 71 71 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 34 34 34 63 63 63 F F F 12 240-259 36 36 36 65 65 65 F F F 13 260-279 44 44 44 81 81 81 F F F 14 280-299 36 36 36 65 65 65 F F F 15 300-319 44 44 44 81 81 81 F F F 16 320-339 43 43 43 76 76 76 F F F 17 340-359 45 45 45 82 82 82 F F F 18 360-379 40 40 40 68 68 68 F F F 19 380-399 42 42 42 72 72 72 F F F 20 400-419 42 42 42 72 72 72 F F F 21 420-439 42 42 42 73 73 73 F F F 22 440-459 45 45 45 82 82 82 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 44 44 44 78 78 78 F F F 25 500-519 44 44 44 78 78 78 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 43 43 43 76 76 76 F F F 28 560-579 43 43 43 76 76 76 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 45, Max flush: 82 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.641646388Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.649735344Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650041018Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650070234Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650078823Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650092412Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317} {"msg":"Crucible 8eacdd90-04b1-463f-b907-c898e5512968 has session id: cfcd21e4-38ca-4622-9dab-91352fb8e5c1","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650442971Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"Upstairs opts: Upstairs UUID: 8eacdd90-04b1-463f-b907-c898e5512968, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650462449Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"Crucible stats registered with UUID: 8eacdd90-04b1-463f-b907-c898e5512968","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650472218Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:53.650481166Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.6506257Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650641328Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650651377Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.65079054Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650802339Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317} {"msg":"8eacdd90-04b1-463f-b907-c898e5512968 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650819027Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650827826Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"client":"0","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650837435Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"client":"1","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650847094Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"client":"2","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650856793Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650868391Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.6508784Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45098)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650953581Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:50605)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.650980288Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:33087)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.651034682Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.651629012Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"client":"1","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.65164746Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"client":"0","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.651729471Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"client":"2","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.651956734Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.651966443Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"[0]R flush_numbers[0..12]: [76, 82, 67, 68, 81, 81, 74, 81, 81, 71, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.65199181Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"downstairs","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"[0]R generation[0..12]: [43, 45, 39, 40, 44, 44, 42, 44, 44, 42, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652002769Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"downstairs","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652013938Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"downstairs","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"[1]R flush_numbers[0..12]: [76, 82, 67, 68, 81, 81, 74, 81, 81, 71, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652024456Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"downstairs","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"[1]R generation[0..12]: [43, 45, 39, 40, 44, 44, 42, 44, 44, 42, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652034695Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"downstairs","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652047654Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"downstairs","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"[2]R flush_numbers[0..12]: [76, 82, 67, 68, 81, 81, 74, 81, 81, 71, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652058072Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"downstairs","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"[2]R generation[0..12]: [43, 45, 39, 40, 44, 44, 42, 44, 44, 42, 34, 34]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652068381Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"downstairs","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.65207942Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"downstairs","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652088789Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"downstairs","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652098528Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"downstairs","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652107747Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"downstairs","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652117046Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"":"downstairs","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652126994Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652135783Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"client":"0","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652145362Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"client":"1","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652154871Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"client":"2","":"downstairs","session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.65216458Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652173699Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"8eacdd90-04b1-463f-b907-c898e5512968 is now active with session: cfcd21e4-38ca-4622-9dab-91352fb8e5c1","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652183598Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652193887Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"session_id":"cfcd21e4-38ca-4622-9dab-91352fb8e5c1"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652202895Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317} {"msg":"Activated sub_volume 8eacdd90-04b1-463f-b907-c898e5512968","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-09-16T04:40:53.652210775Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652224383Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.652469634Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"task":"crutest"} 01/30 Write block 420 len 36864 data: 3 3 2 2 2 2 5 6 5 02/30 Read block 337 len 28672 03/30 Flush 04/30 Write block 431 len 20480 data: 6 7 7 6 5 05/30 Read block 95 len 16384 06/30 Write block 300 len 40960 data: 6 6 7 7 5 7 6 5 5 4 07/30 Read block 596 len 16384 08/30 Flush 09/30 Write block 313 len 40960 data: 2 2 4 5 6 6 7 6 6 3 10/30 Read block 530 len 12288 11/30 Read block 3 len 8192 12/30 Write block 316 len 28672 data: 6 7 7 8 7 7 4 13/30 Read block 29 len 16384 14/30 Read block 496 len 8192 15/30 Read block 94 len 24576 16/30 Read block 49 len 32768 17/30 Flush 18/30 Write block 262 len 36864 data: 4 5 5 4 5 7 6 7 8 19/30 Read block 84 len 24576 20/30 Read block 354 len 24576 21/30 Flush 22/30 Write block 226 len 40960 data: 3 3 3 3 2 2 3 3 3 4 23/30 Write block 293 len 8192 data: 5 5 24/30 Write block 318 len 32768 data: 8 9 8 8 5 3 3 2 25/30 Write block 513 len 8192 data: 3 3 26/30 Read block 503 len 8192 27/30 Read block 296 len 24576 28/30 Write block 400 len 20480 data: 6 6 3 2 2 29/30 Read block 586 len 36864 30/30 Write block 574 len 16384 data: 7 8 6 6 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 5 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Write 10 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Read 3 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 25 27 Done 26 26 2 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:53.66184186Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:53.662011771Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1317,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 76 76 76 F F F 1 020-039 45 45 45 82 82 82 F F F 2 040-059 39 39 39 67 67 67 F F F 3 060-079 40 40 40 68 68 68 F F F 4 080-099 44 44 44 81 81 81 F F F 5 100-119 44 44 44 81 81 81 F F F 6 120-139 42 42 42 74 74 74 F F F 7 140-159 44 44 44 81 81 81 F F F 8 160-179 44 44 44 81 81 81 F F F 9 180-199 42 42 42 71 71 71 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 34 34 34 63 63 63 T T F <--- 12 240-259 36 36 36 65 65 65 F F F 13 260-279 46 46 44 86 86 81 F F F <--- 14 280-299 36 36 36 65 65 65 T T F <--- 15 300-319 46 46 44 85 85 81 T T F <--- 16 320-339 46 46 43 85 85 76 T T F <--- 17 340-359 45 45 45 82 82 82 F F F 18 360-379 40 40 40 68 68 68 F F F 19 380-399 42 42 42 72 72 72 F F F 20 400-419 42 42 42 72 72 72 T T F <--- 21 420-439 46 46 46 84 84 83 F F T <--- 22 440-459 45 45 45 82 82 82 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 44 44 44 78 78 78 F F F 25 500-519 44 44 44 78 78 78 T T F <--- 26 520-539 30 30 30 51 51 51 F F F 27 540-559 43 43 43 76 76 76 F F F 28 560-579 43 43 43 76 76 76 T T F <--- 29 580-599 44 44 44 81 81 81 F F F Max gen: 46, Max flush: 86 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 76 76 76 F F F 1 020-039 45 45 45 82 82 82 F F F 2 040-059 39 39 39 67 67 67 F F F 3 060-079 40 40 40 68 68 68 F F F 4 080-099 44 44 44 81 81 81 F F F 5 100-119 44 44 44 81 81 81 F F F 6 120-139 42 42 42 74 74 74 F F F 7 140-159 44 44 44 81 81 81 F F F 8 160-179 44 44 44 81 81 81 F F F 9 180-199 42 42 42 71 71 71 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 47 47 47 87 87 87 F F F 12 240-259 36 36 36 65 65 65 F F F 13 260-279 46 46 46 86 86 86 F F F 14 280-299 47 47 47 87 87 87 F F F 15 300-319 47 47 47 87 87 87 F F F 16 320-339 47 47 47 87 87 87 F F F 17 340-359 45 45 45 82 82 82 F F F 18 360-379 40 40 40 68 68 68 F F F 19 380-399 42 42 42 72 72 72 F F F 20 400-419 47 47 47 87 87 87 F F F 21 420-439 46 46 46 84 84 84 F F F 22 440-459 45 45 45 82 82 82 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 44 44 44 78 78 78 F F F 25 500-519 47 47 47 87 87 87 F F F 26 520-539 30 30 30 51 51 51 F F F 27 540-559 43 43 43 76 76 76 F F F 28 560-579 47 47 47 87 87 87 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 47, Max flush: 87 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.901002657Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.909129768Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.909403646Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.909433293Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.909441842Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.909453271Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325} {"msg":"Crucible ff221319-706d-45eb-9b93-613419722dc1 has session id: e2add7b4-9c93-4161-aed8-8eb2600c1c10","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.909733788Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"Upstairs opts: Upstairs UUID: ff221319-706d-45eb-9b93-613419722dc1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.909763434Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"Crucible stats registered with UUID: ff221319-706d-45eb-9b93-613419722dc1","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.909774713Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:57.909784202Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.909926615Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.909943943Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.909954082Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.910108664Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.910120803Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325} {"msg":"ff221319-706d-45eb-9b93-613419722dc1 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.910137701Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.91014649Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"client":"0","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.910156248Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"client":"1","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.910165997Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"client":"2","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.910175626Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.910185535Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.910195464Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"ds_connection connected from Ok(127.0.0.1:49135)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.910247158Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"ds_connection connected from Ok(127.0.0.1:37227)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.910272555Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"ds_connection connected from Ok(127.0.0.1:60757)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.910287153Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.910926019Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"client":"2","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.910950316Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"client":"1","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911029347Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"client":"0","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911216545Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911244012Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"[0]R flush_numbers[0..12]: [76, 82, 67, 68, 81, 81, 74, 81, 81, 71, 60, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911303865Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"downstairs","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"[0]R generation[0..12]: [43, 45, 39, 40, 44, 44, 42, 44, 44, 42, 34, 47]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911315093Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"downstairs","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911326092Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"downstairs","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"[1]R flush_numbers[0..12]: [76, 82, 67, 68, 81, 81, 74, 81, 81, 71, 60, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911336521Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"downstairs","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"[1]R generation[0..12]: [43, 45, 39, 40, 44, 44, 42, 44, 44, 42, 34, 47]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911346989Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"downstairs","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911358078Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"downstairs","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"[2]R flush_numbers[0..12]: [76, 82, 67, 68, 81, 81, 74, 81, 81, 71, 60, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911368357Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"downstairs","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"[2]R generation[0..12]: [43, 45, 39, 40, 44, 44, 42, 44, 44, 42, 34, 47]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911378496Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"downstairs","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911389265Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"downstairs","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911398623Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"downstairs","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911408412Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"downstairs","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911419051Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"downstairs","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.91142831Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"":"downstairs","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911438159Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911446878Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"client":"0","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911456587Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"client":"1","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911466136Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"client":"2","":"downstairs","session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911475914Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911484993Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"ff221319-706d-45eb-9b93-613419722dc1 is now active with session: e2add7b4-9c93-4161-aed8-8eb2600c1c10","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911494882Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911503621Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"session_id":"e2add7b4-9c93-4161-aed8-8eb2600c1c10"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.91151276Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325} {"msg":"Activated sub_volume ff221319-706d-45eb-9b93-613419722dc1","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911520709Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911528938Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.911829683Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"task":"crutest"} 01/30 Write block 290 len 8192 data: 5 5 02/30 Read block 333 len 32768 03/30 Read block 293 len 40960 04/30 Read block 125 len 36864 05/30 Read block 40 len 4096 06/30 Write block 530 len 28672 data: 3 3 4 5 5 5 5 07/30 Read block 531 len 28672 08/30 Read block 365 len 4096 09/30 Write block 171 len 24576 data: 4 5 5 4 4 4 10/30 Read block 318 len 20480 11/30 Read block 554 len 16384 12/30 Read block 228 len 12288 13/30 Flush 14/30 Write block 26 len 36864 data: 6 5 6 8 9 10 10 9 6 15/30 Write block 185 len 4096 data: 4 16/30 Read block 112 len 28672 17/30 Read block 215 len 12288 18/30 Write block 47 len 4096 data: 6 19/30 Read block 583 len 36864 20/30 Write block 541 len 36864 data: 3 4 4 5 6 6 6 5 6 21/30 Write block 492 len 4096 data: 7 22/30 Write block 496 len 24576 data: 5 3 3 4 4 2 23/30 Write block 130 len 16384 data: 7 5 4 4 24/30 Read block 512 len 40960 25/30 Read block 182 len 32768 26/30 Read block 298 len 4096 27/30 Write block 386 len 12288 data: 5 5 4 28/30 Write block 375 len 8192 data: 3 3 29/30 Read block 406 len 16384 30/30 Write block 63 len 28672 data: 3 4 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 9 Done Done Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Write 9 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Write 4 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 26 1 28 Done 29 4 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) None Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:40:57.919871974Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:40:57.920042544Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1325,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 76 76 76 F F F 1 020-039 45 45 45 82 82 82 T F T <--- 2 040-059 39 39 39 67 67 67 T F T <--- 3 060-079 40 40 40 68 68 68 T F T <--- 4 080-099 44 44 44 81 81 81 F F F 5 100-119 44 44 44 81 81 81 F F F 6 120-139 42 42 42 74 74 74 T F T <--- 7 140-159 44 44 44 81 81 81 F F F 8 160-179 48 44 48 88 81 88 F F F <--- 9 180-199 42 42 42 71 71 71 T F T <--- 10 200-219 34 34 34 60 60 60 F F F 11 220-239 47 47 47 87 87 87 F F F 12 240-259 36 36 36 65 65 65 F F F 13 260-279 46 46 46 86 86 86 F F F 14 280-299 48 47 48 88 87 88 F T F <--- 15 300-319 47 47 47 87 87 87 F F F 16 320-339 47 47 47 87 87 87 F F F 17 340-359 45 45 45 82 82 82 F F F 18 360-379 40 40 40 68 68 68 T F T <--- 19 380-399 42 42 42 72 72 72 T F T <--- 20 400-419 47 47 47 87 87 87 F F F 21 420-439 46 46 46 84 84 84 F F F 22 440-459 45 45 45 82 82 82 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 44 44 44 78 78 78 T F T <--- 25 500-519 47 47 47 87 87 87 T F T <--- 26 520-539 48 30 48 88 51 88 F F F <--- 27 540-559 43 43 43 76 76 76 T F T <--- 28 560-579 47 47 47 87 87 87 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 48, Max flush: 88 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 76 76 76 F F F 1 020-039 49 49 49 89 89 89 F F F 2 040-059 49 49 49 89 89 89 F F F 3 060-079 49 49 49 89 89 89 F F F 4 080-099 44 44 44 81 81 81 F F F 5 100-119 44 44 44 81 81 81 F F F 6 120-139 49 49 49 89 89 89 F F F 7 140-159 44 44 44 81 81 81 F F F 8 160-179 48 48 48 88 88 88 F F F 9 180-199 49 49 49 89 89 89 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 47 47 47 87 87 87 F F F 12 240-259 36 36 36 65 65 65 F F F 13 260-279 46 46 46 86 86 86 F F F 14 280-299 48 48 48 88 88 88 F F F 15 300-319 47 47 47 87 87 87 F F F 16 320-339 47 47 47 87 87 87 F F F 17 340-359 45 45 45 82 82 82 F F F 18 360-379 49 49 49 89 89 89 F F F 19 380-399 49 49 49 89 89 89 F F F 20 400-419 47 47 47 87 87 87 F F F 21 420-439 46 46 46 84 84 84 F F F 22 440-459 45 45 45 82 82 82 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 49 49 49 89 89 89 F F F 25 500-519 49 49 49 89 89 89 F F F 26 520-539 48 48 48 88 88 88 F F F 27 540-559 49 49 49 89 89 89 F F F 28 560-579 47 47 47 87 87 87 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 49, Max flush: 89 Begin loop 25 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.170659884Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.178710345Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.178995401Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179043476Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179069623Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179082171Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333} {"msg":"Crucible 0897b7b9-5f93-4df0-b482-cacbf7809c23 has session id: 6c72ed7f-df31-4310-af66-4e6ced9d7141","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179414942Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"Upstairs opts: Upstairs UUID: 0897b7b9-5f93-4df0-b482-cacbf7809c23, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179439629Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"Crucible stats registered with UUID: 0897b7b9-5f93-4df0-b482-cacbf7809c23","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179449608Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:02.179458607Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179600201Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179628957Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179640026Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179766591Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179782759Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333} {"msg":"0897b7b9-5f93-4df0-b482-cacbf7809c23 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179819035Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179828644Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"client":"0","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179838283Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"client":"1","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179850102Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"client":"2","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.1798658Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179876158Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179886177Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"ds_connection connected from Ok(127.0.0.1:63124)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.179926033Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"ds_connection connected from Ok(127.0.0.1:38612)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.180080225Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"ds_connection connected from Ok(127.0.0.1:53660)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.180092463Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.180627141Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"client":"0","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.180690523Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"client":"1","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.180779943Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"client":"2","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.180917867Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.18097418Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"[0]R flush_numbers[0..12]: [76, 89, 89, 89, 81, 81, 89, 81, 88, 89, 60, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181037633Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"downstairs","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"[0]R generation[0..12]: [43, 49, 49, 49, 44, 44, 49, 44, 48, 49, 34, 47]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181049871Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"downstairs","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.18106113Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"downstairs","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"[1]R flush_numbers[0..12]: [76, 89, 89, 89, 81, 81, 89, 81, 88, 89, 60, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181074139Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"downstairs","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"[1]R generation[0..12]: [43, 49, 49, 49, 44, 44, 49, 44, 48, 49, 34, 47]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181084447Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"downstairs","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181095326Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"downstairs","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"[2]R flush_numbers[0..12]: [76, 89, 89, 89, 81, 81, 89, 81, 88, 89, 60, 87]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181105755Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"downstairs","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"[2]R generation[0..12]: [43, 49, 49, 49, 44, 44, 49, 44, 48, 49, 34, 47]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181116274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"downstairs","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181126982Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"downstairs","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181136471Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"downstairs","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.18114605Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"downstairs","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181155419Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"downstairs","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181164728Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"":"downstairs","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181174467Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181183166Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"client":"0","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181192905Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"client":"1","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181202634Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"client":"2","":"downstairs","session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181212323Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181222821Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"0897b7b9-5f93-4df0-b482-cacbf7809c23 is now active with session: 6c72ed7f-df31-4310-af66-4e6ced9d7141","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.18123266Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181241399Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"session_id":"6c72ed7f-df31-4310-af66-4e6ced9d7141"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181250458Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333} {"msg":"Activated sub_volume 0897b7b9-5f93-4df0-b482-cacbf7809c23","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181258627Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.181266746Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.18157241Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"task":"crutest"} 01/30 Write block 561 len 32768 data: 4 4 4 6 4 4 4 4 02/30 Flush 03/30 Write block 544 len 28672 data: 6 7 7 7 6 7 7 04/30 Read block 445 len 4096 05/30 Read block 156 len 28672 06/30 Read block 425 len 24576 07/30 Read block 71 len 16384 08/30 Read block 414 len 24576 09/30 Flush 10/30 Flush 11/30 Write block 516 len 20480 data: 4 4 4 4 4 12/30 Write block 150 len 40960 data: 7 6 6 7 6 5 4 4 5 8 13/30 Read block 5 len 36864 14/30 Write block 440 len 32768 data: 3 5 5 6 5 5 5 5 15/30 Write block 317 len 36864 data: 8 9 10 9 9 6 4 4 3 16/30 Read block 548 len 4096 17/30 Read block 442 len 24576 18/30 Write block 176 len 4096 data: 5 19/30 Flush 20/30 Write block 399 len 28672 data: 6 7 7 4 3 3 3 21/30 Read block 33 len 4096 22/30 Flush 23/30 Write block 136 len 28672 data: 3 3 3 2 3 3 4 24/30 Read block 319 len 4096 25/30 Read block 278 len 40960 26/30 Read block 548 len 24576 27/30 Write block 223 len 20480 data: 3 3 3 4 4 28/30 Read block 210 len 24576 29/30 Read block 192 len 4096 30/30 Read block 49 len 24576 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Sent false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 7 Done Done Sent false 1003 Acked Read 1 Done Done Sent false 1004 Acked Read 7 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 9 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Read 6 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:02.191140224Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:02.191313743Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1333,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 76 76 76 F F F 1 020-039 49 49 49 89 89 89 F F F 2 040-059 49 49 49 89 89 89 F F F 3 060-079 49 49 49 89 89 89 F F F 4 080-099 44 44 44 81 81 81 F F F 5 100-119 44 44 44 81 81 81 F F F 6 120-139 49 49 49 89 89 89 T T F <--- 7 140-159 50 50 44 93 93 81 T T F <--- 8 160-179 50 50 48 93 93 88 F F F <--- 9 180-199 49 49 49 89 89 89 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 47 47 47 87 87 87 T T F <--- 12 240-259 36 36 36 65 65 65 F F F 13 260-279 46 46 46 86 86 86 F F F 14 280-299 48 48 48 88 88 88 F F F 15 300-319 50 50 47 93 93 87 F F F <--- 16 320-339 50 50 47 93 93 87 F F F <--- 17 340-359 45 45 45 82 82 82 F F F 18 360-379 49 49 49 89 89 89 F F F 19 380-399 50 50 49 94 94 89 F F F <--- 20 400-419 50 50 47 94 94 87 F F F <--- 21 420-439 46 46 46 84 84 84 F F F 22 440-459 50 50 45 93 93 82 F F F <--- 23 460-479 45 45 45 82 82 82 F F F 24 480-499 49 49 49 89 89 89 F F F 25 500-519 50 50 49 93 93 89 F F F <--- 26 520-539 50 50 48 93 93 88 F F F <--- 27 540-559 50 50 49 91 91 89 F F F <--- 28 560-579 50 50 47 90 90 87 F F F <--- 29 580-599 44 44 44 81 81 81 F F F Max gen: 50, Max flush: 94 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 76 76 76 F F F 1 020-039 49 49 49 89 89 89 F F F 2 040-059 49 49 49 89 89 89 F F F 3 060-079 49 49 49 89 89 89 F F F 4 080-099 44 44 44 81 81 81 F F F 5 100-119 44 44 44 81 81 81 F F F 6 120-139 51 51 51 95 95 95 F F F 7 140-159 51 51 51 95 95 95 F F F 8 160-179 50 50 50 93 93 93 F F F 9 180-199 49 49 49 89 89 89 F F F 10 200-219 34 34 34 60 60 60 F F F 11 220-239 51 51 51 95 95 95 F F F 12 240-259 36 36 36 65 65 65 F F F 13 260-279 46 46 46 86 86 86 F F F 14 280-299 48 48 48 88 88 88 F F F 15 300-319 50 50 50 93 93 93 F F F 16 320-339 50 50 50 93 93 93 F F F 17 340-359 45 45 45 82 82 82 F F F 18 360-379 49 49 49 89 89 89 F F F 19 380-399 50 50 50 94 94 94 F F F 20 400-419 50 50 50 94 94 94 F F F 21 420-439 46 46 46 84 84 84 F F F 22 440-459 50 50 50 93 93 93 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 49 49 49 89 89 89 F F F 25 500-519 50 50 50 93 93 93 F F F 26 520-539 50 50 50 93 93 93 F F F 27 540-559 50 50 50 91 91 91 F F F 28 560-579 50 50 50 90 90 90 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 51, Max flush: 95 Begin loop 26 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.436256596Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.444403125Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.444676363Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.44470256Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.444711359Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.444723127Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341} {"msg":"Crucible 9c66a4ff-f284-47d4-bcf9-1080d2879e7d has session id: bc6e5cd8-b632-4297-bb8e-e079b41d2b12","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445076646Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"Upstairs opts: Upstairs UUID: 9c66a4ff-f284-47d4-bcf9-1080d2879e7d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445097004Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"Crucible stats registered with UUID: 9c66a4ff-f284-47d4-bcf9-1080d2879e7d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445108882Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:06.445118061Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445266254Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445295181Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445306089Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445426415Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445442413Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341} {"msg":"9c66a4ff-f284-47d4-bcf9-1080d2879e7d active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445477739Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445487218Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"client":"0","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445496867Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"client":"1","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445506726Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"client":"2","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445516185Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445526114Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445535942Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"ds_connection connected from Ok(127.0.0.1:60932)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445581437Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"ds_connection connected from Ok(127.0.0.1:36281)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445592386Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"ds_connection connected from Ok(127.0.0.1:63973)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.445705453Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446296374Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"client":"1","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446320911Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"client":"0","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446332639Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"client":"2","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446612127Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446622016Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"[0]R flush_numbers[0..12]: [76, 89, 89, 89, 81, 81, 95, 95, 93, 89, 60, 95]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446654722Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"downstairs","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"[0]R generation[0..12]: [43, 49, 49, 49, 44, 44, 51, 51, 50, 49, 34, 51]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446665781Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"downstairs","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446676709Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"downstairs","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"[1]R flush_numbers[0..12]: [76, 89, 89, 89, 81, 81, 95, 95, 93, 89, 60, 95]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446687218Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"downstairs","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"[1]R generation[0..12]: [43, 49, 49, 49, 44, 44, 51, 51, 50, 49, 34, 51]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446697567Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"downstairs","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446708406Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"downstairs","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"[2]R flush_numbers[0..12]: [76, 89, 89, 89, 81, 81, 95, 95, 93, 89, 60, 95]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446718814Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"downstairs","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"[2]R generation[0..12]: [43, 49, 49, 49, 44, 44, 51, 51, 50, 49, 34, 51]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446729063Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"downstairs","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446739982Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"downstairs","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.44675226Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"downstairs","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446762099Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"downstairs","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446771548Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"downstairs","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446781047Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"":"downstairs","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446790626Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446799395Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"client":"0","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446808944Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"client":"1","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446818663Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"client":"2","":"downstairs","session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446828372Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.4468375Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"9c66a4ff-f284-47d4-bcf9-1080d2879e7d is now active with session: bc6e5cd8-b632-4297-bb8e-e079b41d2b12","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446847719Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446856648Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"session_id":"bc6e5cd8-b632-4297-bb8e-e079b41d2b12"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446865767Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341} {"msg":"Activated sub_volume 9c66a4ff-f284-47d4-bcf9-1080d2879e7d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446873876Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.446881975Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.447154104Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"task":"crutest"} 01/30 Write block 549 len 24576 data: 8 8 6 6 7 6 02/30 Write block 506 len 24576 data: 3 3 3 2 3 3 03/30 Flush 04/30 Flush 05/30 Write block 310 len 4096 data: 3 06/30 Write block 515 len 24576 data: 4 5 5 5 5 5 07/30 Flush 08/30 Write block 381 len 12288 data: 4 6 7 09/30 Write block 444 len 28672 data: 6 6 6 6 6 4 5 10/30 Write block 539 len 24576 data: 4 3 4 5 5 7 11/30 Write block 490 len 36864 data: 5 4 8 7 7 7 6 4 4 12/30 Write block 507 len 36864 data: 4 4 3 4 4 3 4 4 5 13/30 Write block 250 len 16384 data: 6 7 7 5 14/30 Flush 15/30 Write block 339 len 24576 data: 5 5 4 4 4 4 16/30 Write block 547 len 28672 data: 8 7 9 9 7 7 8 17/30 Write block 202 len 4096 data: 5 18/30 Flush 19/30 Flush 20/30 Write block 200 len 12288 data: 4 4 6 21/30 Flush 22/30 Write block 500 len 12288 data: 5 3 2 23/30 Write block 289 len 28672 data: 6 6 6 4 6 6 5 24/30 Write block 140 len 4096 data: 4 25/30 Write block 180 len 20480 data: 3 4 3 4 3 26/30 Read block 60 len 28672 27/30 Read block 414 len 16384 28/30 Read block 369 len 40960 29/30 Read block 62 len 4096 30/30 Read block 222 len 16384 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 3 Done Done Done false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 10 11 Done 11 10 1 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) Some(JobId(1018)) Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:06.454508505Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:06.454677305Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1341,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 76 76 76 F F F 1 020-039 49 49 49 89 89 89 F F F 2 040-059 49 49 49 89 89 89 F F F 3 060-079 49 49 49 89 89 89 F F F 4 080-099 44 44 44 81 81 81 F F F 5 100-119 44 44 44 81 81 81 F F F 6 120-139 51 51 51 95 95 95 F F F 7 140-159 51 51 51 95 95 95 T T F <--- 8 160-179 50 50 50 93 93 93 F F F 9 180-199 49 49 49 89 89 89 T T F <--- 10 200-219 52 52 52 102 102 100 F F T <--- 11 220-239 51 51 51 95 95 95 F F F 12 240-259 52 52 52 99 99 99 F F F 13 260-279 46 46 46 86 86 86 F F F 14 280-299 48 48 48 88 88 88 T T F <--- 15 300-319 52 52 52 98 98 98 F F F 16 320-339 52 52 52 100 100 100 F F F 17 340-359 52 52 52 100 100 100 F F F 18 360-379 49 49 49 89 89 89 F F F 19 380-399 52 52 52 99 99 99 F F F 20 400-419 50 50 50 94 94 94 F F F 21 420-439 46 46 46 84 84 84 F F F 22 440-459 52 52 52 99 99 99 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 52 52 52 99 99 99 F F F 25 500-519 52 52 52 99 99 99 T T F <--- 26 520-539 52 52 52 99 99 99 F F F 27 540-559 52 52 52 100 100 100 F F F 28 560-579 50 50 50 90 90 90 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 52, Max flush: 102 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 76 76 76 F F F 1 020-039 49 49 49 89 89 89 F F F 2 040-059 49 49 49 89 89 89 F F F 3 060-079 49 49 49 89 89 89 F F F 4 080-099 44 44 44 81 81 81 F F F 5 100-119 44 44 44 81 81 81 F F F 6 120-139 51 51 51 95 95 95 F F F 7 140-159 53 53 53 103 103 103 F F F 8 160-179 50 50 50 93 93 93 F F F 9 180-199 53 53 53 103 103 103 F F F 10 200-219 52 52 52 102 102 102 F F F 11 220-239 51 51 51 95 95 95 F F F 12 240-259 52 52 52 99 99 99 F F F 13 260-279 46 46 46 86 86 86 F F F 14 280-299 53 53 53 103 103 103 F F F 15 300-319 52 52 52 98 98 98 F F F 16 320-339 52 52 52 100 100 100 F F F 17 340-359 52 52 52 100 100 100 F F F 18 360-379 49 49 49 89 89 89 F F F 19 380-399 52 52 52 99 99 99 F F F 20 400-419 50 50 50 94 94 94 F F F 21 420-439 46 46 46 84 84 84 F F F 22 440-459 52 52 52 99 99 99 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 52 52 52 99 99 99 F F F 25 500-519 53 53 53 103 103 103 F F F 26 520-539 52 52 52 99 99 99 F F F 27 540-559 52 52 52 100 100 100 F F F 28 560-579 50 50 50 90 90 90 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 53, Max flush: 103 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.68632255Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:10.69351348Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.693550556Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.700607932Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.700878451Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.700903428Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.700914676Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.700926215Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349} {"msg":"Crucible d38b59c4-6991-4179-b4e7-c03599ca67a9 has session id: a711cf2d-bd86-488a-b9df-2447ac79e3ce","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701284393Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"Upstairs opts: Upstairs UUID: d38b59c4-6991-4179-b4e7-c03599ca67a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701305171Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"Crucible stats registered with UUID: d38b59c4-6991-4179-b4e7-c03599ca67a9","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.70131497Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:10.701324159Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701466052Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.7014808Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701491119Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701628243Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701644831Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349} {"msg":"d38b59c4-6991-4179-b4e7-c03599ca67a9 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701677287Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701686726Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"client":"0","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701696505Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"client":"1","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701708004Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"client":"2","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701717853Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701727792Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.70173768Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:33892)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701810832Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:41341)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701835489Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:55570)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.701887823Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.702442308Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"client":"1","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.702458486Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"client":"0","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.70259678Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"client":"2","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.702759121Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.702774889Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"[0]R flush_numbers[0..12]: [76, 89, 89, 89, 81, 81, 95, 103, 93, 103, 102, 95]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.702907864Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"downstairs","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"[0]R generation[0..12]: [43, 49, 49, 49, 44, 44, 51, 53, 50, 53, 52, 51]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.702919222Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"downstairs","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.702930171Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"downstairs","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"[1]R flush_numbers[0..12]: [76, 89, 89, 89, 81, 81, 95, 103, 93, 103, 102, 95]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.70294317Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"downstairs","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"[1]R generation[0..12]: [43, 49, 49, 49, 44, 44, 51, 53, 50, 53, 52, 51]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.702953798Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"downstairs","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.702964597Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"downstairs","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"[2]R flush_numbers[0..12]: [76, 89, 89, 89, 81, 81, 95, 103, 93, 103, 102, 95]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.702974996Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"downstairs","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"[2]R generation[0..12]: [43, 49, 49, 49, 44, 44, 51, 53, 50, 53, 52, 51]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.702985205Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"downstairs","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.702996203Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"downstairs","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703005842Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"downstairs","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703015481Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"downstairs","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.70302467Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"downstairs","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703034099Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"":"downstairs","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703043828Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703052617Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"client":"0","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703062346Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"client":"1","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703071835Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"client":"2","":"downstairs","session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703083243Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703092382Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"d38b59c4-6991-4179-b4e7-c03599ca67a9 is now active with session: a711cf2d-bd86-488a-b9df-2447ac79e3ce","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703102301Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.70311103Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"session_id":"a711cf2d-bd86-488a-b9df-2447ac79e3ce"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703120369Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349} {"msg":"Activated sub_volume d38b59c4-6991-4179-b4e7-c03599ca67a9","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703128358Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703136477Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.703426433Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"task":"crutest"} 01/30 Write block 544 len 40960 data: 8 8 8 9 8 10 10 8 8 9 02/30 Read block 394 len 40960 03/30 Flush 04/30 Write block 565 len 16384 data: 5 5 5 5 05/30 Flush 06/30 Write block 226 len 32768 data: 5 5 4 4 3 3 4 4 07/30 Write block 135 len 20480 data: 4 4 4 4 3 08/30 Read block 69 len 12288 09/30 Read block 289 len 40960 10/30 Write block 237 len 32768 data: 5 6 7 5 5 6 6 7 11/30 Read block 435 len 24576 12/30 Read block 370 len 4096 13/30 Read block 144 len 20480 14/30 Write block 197 len 40960 data: 5 6 4 5 5 7 5 5 4 4 15/30 Write block 90 len 20480 data: 8 6 7 6 6 16/30 Read block 119 len 24576 17/30 Read block 508 len 20480 18/30 Read block 584 len 4096 19/30 Write block 65 len 4096 data: 4 20/30 Read block 125 len 16384 21/30 Read block 119 len 40960 22/30 Flush 23/30 Write block 220 len 24576 data: 3 3 3 4 4 4 24/30 Write block 435 len 8192 data: 6 6 25/30 Write block 439 len 36864 data: 3 4 6 6 7 7 7 7 7 26/30 Read block 552 len 20480 27/30 Write block 319 len 4096 data: 11 28/30 Write block 278 len 28672 data: 10 11 8 6 5 3 3 29/30 Read block 293 len 4096 30/30 Read block 319 len 4096 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Write 5 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 5 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 0 0 25 Done 0 25 25 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:10.712918635Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:10.713077717Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1349,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 76 76 76 F F F 1 020-039 49 49 49 89 89 89 F F F 2 040-059 49 49 49 89 89 89 F F F 3 060-079 49 54 54 89 106 106 F F F <--- 4 080-099 44 54 54 81 106 106 F F F <--- 5 100-119 44 44 44 81 81 81 F F F 6 120-139 51 54 54 95 106 106 F F F <--- 7 140-159 53 53 53 103 103 103 F F F 8 160-179 50 50 50 93 93 93 F F F 9 180-199 53 54 54 103 106 106 F F F <--- 10 200-219 52 54 54 102 106 106 F F F <--- 11 220-239 51 54 54 95 106 106 F T T <--- 12 240-259 52 54 54 99 106 106 F F F <--- 13 260-279 46 46 46 86 86 86 F T T <--- 14 280-299 53 53 53 103 103 103 F T T <--- 15 300-319 52 52 52 98 98 98 F T T <--- 16 320-339 52 52 52 100 100 100 F F F 17 340-359 52 52 52 100 100 100 F F F 18 360-379 49 49 49 89 89 89 F F F 19 380-399 52 52 52 99 99 99 F F F 20 400-419 50 50 50 94 94 94 F F F 21 420-439 46 46 46 84 84 84 F T T <--- 22 440-459 52 52 52 99 99 99 F T T <--- 23 460-479 45 45 45 82 82 82 F F F 24 480-499 52 52 52 99 99 99 F F F 25 500-519 53 53 53 103 103 103 F F F 26 520-539 52 52 52 99 99 99 F F F 27 540-559 54 54 54 104 104 104 F F F 28 560-579 54 54 54 105 105 105 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 54, Max flush: 106 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 43 43 43 76 76 76 F F F 1 020-039 49 49 49 89 89 89 F F F 2 040-059 49 49 49 89 89 89 F F F 3 060-079 54 54 54 106 106 106 F F F 4 080-099 54 54 54 106 106 106 F F F 5 100-119 44 44 44 81 81 81 F F F 6 120-139 54 54 54 106 106 106 F F F 7 140-159 53 53 53 103 103 103 F F F 8 160-179 50 50 50 93 93 93 F F F 9 180-199 54 54 54 106 106 106 F F F 10 200-219 54 54 54 106 106 106 F F F 11 220-239 55 55 55 107 107 107 F F F 12 240-259 54 54 54 106 106 106 F F F 13 260-279 55 55 55 107 107 107 F F F 14 280-299 55 55 55 107 107 107 F F F 15 300-319 55 55 55 107 107 107 F F F 16 320-339 52 52 52 100 100 100 F F F 17 340-359 52 52 52 100 100 100 F F F 18 360-379 49 49 49 89 89 89 F F F 19 380-399 52 52 52 99 99 99 F F F 20 400-419 50 50 50 94 94 94 F F F 21 420-439 55 55 55 107 107 107 F F F 22 440-459 55 55 55 107 107 107 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 52 52 52 99 99 99 F F F 25 500-519 53 53 53 103 103 103 F F F 26 520-539 52 52 52 99 99 99 F F F 27 540-559 54 54 54 104 104 104 F F F 28 560-579 54 54 54 105 105 105 F F F 29 580-599 44 44 44 81 81 81 F F F Max gen: 55, Max flush: 107 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.971280001Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.979716476Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980011681Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980037278Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980045797Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980057456Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357} {"msg":"Crucible ba55e4b5-38b7-4059-8a1c-bb6f394622a2 has session id: 1b767770-60ea-412a-aa21-bffa5d11496d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980372179Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"Upstairs opts: Upstairs UUID: ba55e4b5-38b7-4059-8a1c-bb6f394622a2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980429673Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"Crucible stats registered with UUID: ba55e4b5-38b7-4059-8a1c-bb6f394622a2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980442071Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:14.98045124Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980593073Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.9806227Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980633919Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980765723Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980800229Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357} {"msg":"ba55e4b5-38b7-4059-8a1c-bb6f394622a2 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980809198Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980818097Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"client":"0","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980827746Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"client":"1","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980837405Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"client":"2","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980846984Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980857013Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980866802Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43503)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980938963Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"ds_connection connected from Ok(127.0.0.1:60451)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.980975219Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59629)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.981023953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.981636002Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"client":"2","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.98164868Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"client":"0","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.981659779Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"client":"1","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.981943606Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.981984081Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"[0]R flush_numbers[0..12]: [76, 89, 89, 106, 106, 81, 106, 103, 93, 106, 106, 107]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982038105Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"downstairs","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"[0]R generation[0..12]: [43, 49, 49, 54, 54, 44, 54, 53, 50, 54, 54, 55]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982049303Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"downstairs","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982060352Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"downstairs","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"[1]R flush_numbers[0..12]: [76, 89, 89, 106, 106, 81, 106, 103, 93, 106, 106, 107]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982070841Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"downstairs","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"[1]R generation[0..12]: [43, 49, 49, 54, 54, 44, 54, 53, 50, 54, 54, 55]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.98208117Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"downstairs","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982092048Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"downstairs","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"[2]R flush_numbers[0..12]: [76, 89, 89, 106, 106, 81, 106, 103, 93, 106, 106, 107]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982102567Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"downstairs","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"[2]R generation[0..12]: [43, 49, 49, 54, 54, 44, 54, 53, 50, 54, 54, 55]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982112856Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"downstairs","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982123795Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"downstairs","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982134694Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"downstairs","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982144372Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"downstairs","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"Next flush: 108","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982153841Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"downstairs","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.98216329Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"":"downstairs","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982173049Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982181738Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"client":"0","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982191357Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"client":"1","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982201046Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"client":"2","":"downstairs","session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982211105Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982220214Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"ba55e4b5-38b7-4059-8a1c-bb6f394622a2 is now active with session: 1b767770-60ea-412a-aa21-bffa5d11496d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982230422Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982239211Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"session_id":"1b767770-60ea-412a-aa21-bffa5d11496d"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.9822483Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357} {"msg":"Activated sub_volume ba55e4b5-38b7-4059-8a1c-bb6f394622a2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982256659Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982269958Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.982572242Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"task":"crutest"} 01/30 Write block 530 len 16384 data: 4 4 5 6 02/30 Read block 276 len 24576 03/30 Flush 04/30 Write block 119 len 16384 data: 5 3 3 3 05/30 Read block 538 len 24576 06/30 Read block 98 len 12288 07/30 Read block 186 len 16384 08/30 Write block 12 len 24576 data: 6 7 6 4 4 4 09/30 Read block 58 len 20480 10/30 Read block 534 len 8192 11/30 Read block 248 len 16384 12/30 Flush 13/30 Flush 14/30 Write block 294 len 28672 data: 7 6 5 4 4 6 7 15/30 Write block 31 len 24576 data: 11 11 10 7 6 5 16/30 Flush 17/30 Write block 114 len 36864 data: 4 4 5 5 6 6 4 4 4 18/30 Read block 256 len 32768 19/30 Flush 20/30 Write block 294 len 40960 data: 8 7 6 5 5 7 8 7 8 8 21/30 Write block 22 len 32768 data: 4 4 4 4 7 6 7 9 22/30 Write block 179 len 8192 data: 3 4 23/30 Read block 16 len 32768 24/30 Read block 435 len 4096 25/30 Read block 550 len 4096 26/30 Write block 582 len 24576 data: 6 7 5 6 5 5 27/30 Read block 199 len 28672 28/30 Read block 69 len 12288 29/30 Write block 204 len 28672 data: 6 5 5 4 5 6 3 30/30 Write block 28 len 20480 data: 8 10 10 12 12 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 6 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 4 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 29 33 Done 28 28 1 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:14.991490441Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:14.991625106Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1357,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 43 109 109 76 F F F <--- 1 020-039 56 56 49 111 111 89 T T F <--- 2 040-059 49 49 49 89 89 89 F F F 3 060-079 54 54 54 106 106 106 F F F 4 080-099 54 54 54 106 106 106 F F F 5 100-119 56 56 44 112 112 81 F F F <--- 6 120-139 56 56 54 112 112 106 F F F <--- 7 140-159 53 53 53 103 103 103 F F F 8 160-179 50 50 50 93 93 93 T T F <--- 9 180-199 54 54 54 106 106 106 T T F <--- 10 200-219 54 54 54 106 106 106 T T F <--- 11 220-239 55 55 55 107 107 107 F F F 12 240-259 54 54 54 106 106 106 F F F 13 260-279 55 55 55 107 107 107 F F F 14 280-299 56 56 55 111 111 107 T T F <--- 15 300-319 56 56 55 111 111 107 T T F <--- 16 320-339 52 52 52 100 100 100 F F F 17 340-359 52 52 52 100 100 100 F F F 18 360-379 49 49 49 89 89 89 F F F 19 380-399 52 52 52 99 99 99 F F F 20 400-419 50 50 50 94 94 94 F F F 21 420-439 55 55 55 107 107 107 F F F 22 440-459 55 55 55 107 107 107 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 52 52 52 99 99 99 F F F 25 500-519 53 53 53 103 103 103 F F F 26 520-539 56 56 52 108 108 99 F F T <--- 27 540-559 54 54 54 104 104 104 F F F 28 560-579 54 54 54 105 105 105 F F F 29 580-599 44 44 44 81 81 81 T T F <--- Max gen: 56, Max flush: 112 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 109 109 109 F F F 1 020-039 57 57 57 113 113 113 F F F 2 040-059 49 49 49 89 89 89 F F F 3 060-079 54 54 54 106 106 106 F F F 4 080-099 54 54 54 106 106 106 F F F 5 100-119 56 56 56 112 112 112 F F F 6 120-139 56 56 56 112 112 112 F F F 7 140-159 53 53 53 103 103 103 F F F 8 160-179 57 57 57 113 113 113 F F F 9 180-199 57 57 57 113 113 113 F F F 10 200-219 57 57 57 113 113 113 F F F 11 220-239 55 55 55 107 107 107 F F F 12 240-259 54 54 54 106 106 106 F F F 13 260-279 55 55 55 107 107 107 F F F 14 280-299 57 57 57 113 113 113 F F F 15 300-319 57 57 57 113 113 113 F F F 16 320-339 52 52 52 100 100 100 F F F 17 340-359 52 52 52 100 100 100 F F F 18 360-379 49 49 49 89 89 89 F F F 19 380-399 52 52 52 99 99 99 F F F 20 400-419 50 50 50 94 94 94 F F F 21 420-439 55 55 55 107 107 107 F F F 22 440-459 55 55 55 107 107 107 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 52 52 52 99 99 99 F F F 25 500-519 53 53 53 103 103 103 F F F 26 520-539 56 56 56 108 108 108 F F F 27 540-559 54 54 54 104 104 104 F F F 28 560-579 54 54 54 105 105 105 F F F 29 580-599 57 57 57 113 113 113 F F F Max gen: 57, Max flush: 113 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.236192362Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.244398804Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.24468928Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.244726786Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.244736534Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.244748333Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365} {"msg":"Crucible 31a66905-4f39-4777-9b12-ba3b841dfdd4 has session id: 98bbc597-b9f5-4ed3-9a03-625473c260bf","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245088253Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"Upstairs opts: Upstairs UUID: 31a66905-4f39-4777-9b12-ba3b841dfdd4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.24511683Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"Crucible stats registered with UUID: 31a66905-4f39-4777-9b12-ba3b841dfdd4","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245128229Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:19.245137658Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245261163Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.24529179Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245302918Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245419905Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245435273Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365} {"msg":"31a66905-4f39-4777-9b12-ba3b841dfdd4 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245470659Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245480278Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"client":"0","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245490056Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"client":"1","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245499765Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"client":"2","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245509274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245519463Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:62246)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245566807Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245577636Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:52476)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.24562921Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:62041)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.245648248Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246253467Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"client":"0","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246300332Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"client":"1","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246351896Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"client":"2","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.24657742Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246587518Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"[0]R flush_numbers[0..12]: [109, 113, 89, 106, 106, 112, 112, 103, 113, 113, 113, 107]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246627084Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"downstairs","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"[0]R generation[0..12]: [56, 57, 49, 54, 54, 56, 56, 53, 57, 57, 57, 55]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246638402Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"downstairs","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246649341Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"downstairs","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"[1]R flush_numbers[0..12]: [109, 113, 89, 106, 106, 112, 112, 103, 113, 113, 113, 107]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.24665992Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"downstairs","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"[1]R generation[0..12]: [56, 57, 49, 54, 54, 56, 56, 53, 57, 57, 57, 55]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246670319Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"downstairs","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246681227Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"downstairs","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"[2]R flush_numbers[0..12]: [109, 113, 89, 106, 106, 112, 112, 103, 113, 113, 113, 107]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246691806Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"downstairs","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"[2]R generation[0..12]: [56, 57, 49, 54, 54, 56, 56, 53, 57, 57, 57, 55]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246702085Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"downstairs","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246713094Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"downstairs","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246722393Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"downstairs","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246732001Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"downstairs","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.24674132Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"downstairs","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246750659Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"":"downstairs","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246760368Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246769097Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"client":"0","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246779886Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"client":"1","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246789505Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"client":"2","":"downstairs","session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246799324Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246808493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"31a66905-4f39-4777-9b12-ba3b841dfdd4 is now active with session: 98bbc597-b9f5-4ed3-9a03-625473c260bf","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246818511Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.24682723Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"session_id":"98bbc597-b9f5-4ed3-9a03-625473c260bf"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246836319Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365} {"msg":"Activated sub_volume 31a66905-4f39-4777-9b12-ba3b841dfdd4","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246844308Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.246852387Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.247149643Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"task":"crutest"} 01/30 Write block 577 len 4096 data: 7 02/30 Read block 212 len 36864 03/30 Write block 26 len 40960 data: 8 7 9 11 11 13 13 11 8 7 04/30 Read block 54 len 40960 05/30 Write block 164 len 24576 data: 6 5 5 5 6 5 06/30 Read block 242 len 40960 07/30 Write block 332 len 16384 data: 6 8 6 6 08/30 Read block 173 len 20480 09/30 Write block 280 len 32768 data: 9 7 6 4 4 4 5 6 10/30 Write block 152 len 4096 data: 7 11/30 Write block 493 len 4096 data: 8 12/30 Read block 48 len 24576 13/30 Write block 394 len 28672 data: 4 4 4 4 6 7 8 14/30 Write block 420 len 36864 data: 4 4 3 3 3 3 6 7 6 15/30 Write block 33 len 40960 data: 12 9 8 6 3 3 3 4 4 4 16/30 Write block 194 len 20480 data: 5 5 5 6 7 17/30 Write block 375 len 8192 data: 4 4 18/30 Write block 427 len 32768 data: 8 7 6 6 7 8 8 7 19/30 Write block 293 len 24576 data: 7 9 8 7 6 6 20/30 Write block 51 len 32768 data: 7 6 5 5 5 6 6 5 21/30 Read block 481 len 36864 22/30 Write block 23 len 40960 data: 5 5 5 9 8 10 12 12 14 14 23/30 Read block 429 len 40960 24/30 Write block 576 len 24576 data: 7 8 4 6 6 6 25/30 Read block 88 len 24576 26/30 Write block 73 len 16384 data: 7 6 4 3 27/30 Read block 194 len 20480 28/30 Write block 246 len 32768 data: 6 4 5 5 7 8 8 6 29/30 Write block 238 len 28672 data: 7 8 6 6 7 7 8 30/30 Read block 160 len 28672 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Sent Done false 1001 Acked Read 9 Done Sent Done false 1002 Acked Write 10 Done Sent Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Write 7 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Read 6 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Write 8 Done Sent Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 0 31 Done 29 0 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:19.255554932Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:19.255730351Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1365,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 109 109 109 F F F 1 020-039 57 57 57 113 113 113 T F T <--- 2 040-059 49 49 49 89 89 89 T F T <--- 3 060-079 54 54 54 106 106 106 T F T <--- 4 080-099 54 54 54 106 106 106 F F F 5 100-119 56 56 56 112 112 112 F F F 6 120-139 56 56 56 112 112 112 F F F 7 140-159 53 53 53 103 103 103 T F T <--- 8 160-179 57 57 57 113 113 113 T F T <--- 9 180-199 57 57 57 113 113 113 T F T <--- 10 200-219 57 57 57 113 113 113 F F F 11 220-239 55 55 55 107 107 107 T F T <--- 12 240-259 54 54 54 106 106 106 T F T <--- 13 260-279 55 55 55 107 107 107 F F F 14 280-299 57 57 57 113 113 113 T F T <--- 15 300-319 57 57 57 113 113 113 F F F 16 320-339 52 52 52 100 100 100 T F T <--- 17 340-359 52 52 52 100 100 100 F F F 18 360-379 49 49 49 89 89 89 T F T <--- 19 380-399 52 52 52 99 99 99 T F T <--- 20 400-419 50 50 50 94 94 94 T F T <--- 21 420-439 55 55 55 107 107 107 T F T <--- 22 440-459 55 55 55 107 107 107 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 52 52 52 99 99 99 T F T <--- 25 500-519 53 53 53 103 103 103 F F F 26 520-539 56 56 56 108 108 108 F F F 27 540-559 54 54 54 104 104 104 F F F 28 560-579 54 54 54 105 105 105 T F T <--- 29 580-599 57 57 57 113 113 113 T F T <--- Max gen: 57, Max flush: 113 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 109 109 109 F F F 1 020-039 58 58 58 114 114 114 F F F 2 040-059 58 58 58 114 114 114 F F F 3 060-079 58 58 58 114 114 114 F F F 4 080-099 54 54 54 106 106 106 F F F 5 100-119 56 56 56 112 112 112 F F F 6 120-139 56 56 56 112 112 112 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 58 58 58 114 114 114 F F F 9 180-199 58 58 58 114 114 114 F F F 10 200-219 57 57 57 113 113 113 F F F 11 220-239 58 58 58 114 114 114 F F F 12 240-259 58 58 58 114 114 114 F F F 13 260-279 55 55 55 107 107 107 F F F 14 280-299 58 58 58 114 114 114 F F F 15 300-319 57 57 57 113 113 113 F F F 16 320-339 58 58 58 114 114 114 F F F 17 340-359 52 52 52 100 100 100 F F F 18 360-379 58 58 58 114 114 114 F F F 19 380-399 58 58 58 114 114 114 F F F 20 400-419 58 58 58 114 114 114 F F F 21 420-439 58 58 58 114 114 114 F F F 22 440-459 55 55 55 107 107 107 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 58 58 58 114 114 114 F F F 25 500-519 53 53 53 103 103 103 F F F 26 520-539 56 56 56 108 108 108 F F F 27 540-559 54 54 54 104 104 104 F F F 28 560-579 58 58 58 114 114 114 F F F 29 580-599 58 58 58 114 114 114 F F F Max gen: 58, Max flush: 114 Begin loop 30 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.518551006Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.526675988Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.526954266Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.52700434Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527032016Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527044815Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373} {"msg":"Crucible 5ec1f5b4-6fce-4518-b6a7-7f3776a92240 has session id: 9d7d8138-59f1-413c-83ad-4593fdbf8b18","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527291066Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"Upstairs opts: Upstairs UUID: 5ec1f5b4-6fce-4518-b6a7-7f3776a92240, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527355349Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"Crucible stats registered with UUID: 5ec1f5b4-6fce-4518-b6a7-7f3776a92240","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527365958Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:23.527374906Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.52751274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527541977Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527552996Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527660683Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527678621Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373} {"msg":"5ec1f5b4-6fce-4518-b6a7-7f3776a92240 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527704808Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527714387Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"client":"0","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527724166Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"client":"1","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527733875Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"client":"2","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527745103Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527755152Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527765061Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"ds_connection connected from Ok(127.0.0.1:39991)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527808456Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"ds_connection connected from Ok(127.0.0.1:39598)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527825244Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"ds_connection connected from Ok(127.0.0.1:57479)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.527838102Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.528553609Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"client":"1","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.528574037Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"client":"0","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.528585325Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"client":"2","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.528830197Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.528852364Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"[0]R flush_numbers[0..12]: [109, 114, 114, 114, 106, 112, 112, 114, 114, 114, 113, 114]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.528919366Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"downstairs","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"[0]R generation[0..12]: [56, 58, 58, 58, 54, 56, 56, 58, 58, 58, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.528931015Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"downstairs","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.528942014Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"downstairs","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"[1]R flush_numbers[0..12]: [109, 114, 114, 114, 106, 112, 112, 114, 114, 114, 113, 114]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.528952722Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"downstairs","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"[1]R generation[0..12]: [56, 58, 58, 58, 54, 56, 56, 58, 58, 58, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.528964821Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"downstairs","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.52897588Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"downstairs","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"[2]R flush_numbers[0..12]: [109, 114, 114, 114, 106, 112, 112, 114, 114, 114, 113, 114]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.528986608Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"downstairs","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"[2]R generation[0..12]: [56, 58, 58, 58, 54, 56, 56, 58, 58, 58, 57, 58]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.528997077Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"downstairs","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529008056Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"downstairs","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"Max found gen is 59","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529017575Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"downstairs","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"Generation requested: 60 >= found:59","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529027034Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"downstairs","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"Next flush: 115","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529036313Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"downstairs","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529045591Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"":"downstairs","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.52905553Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529064349Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"client":"0","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529074128Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"client":"1","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529083967Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"client":"2","":"downstairs","session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529093766Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529102905Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"5ec1f5b4-6fce-4518-b6a7-7f3776a92240 is now active with session: 9d7d8138-59f1-413c-83ad-4593fdbf8b18","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529114123Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529122842Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"session_id":"9d7d8138-59f1-413c-83ad-4593fdbf8b18"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529131811Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373} {"msg":"Activated sub_volume 5ec1f5b4-6fce-4518-b6a7-7f3776a92240","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.52913979Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.52914788Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.529425827Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"task":"crutest"} 01/30 Write block 392 len 36864 data: 3 4 5 5 5 5 7 8 9 02/30 Read block 271 len 32768 03/30 Read block 91 len 4096 04/30 Write block 98 len 4096 data: 5 05/30 Flush 06/30 Write block 225 len 8192 data: 5 6 07/30 Read block 341 len 36864 08/30 Read block 541 len 8192 09/30 Write block 414 len 40960 data: 3 2 3 2 3 3 5 5 4 4 10/30 Read block 81 len 28672 11/30 Flush 12/30 Flush 13/30 Write block 123 len 20480 data: 6 7 7 8 7 14/30 Read block 144 len 8192 15/30 Write block 297 len 12288 data: 7 7 8 16/30 Read block 218 len 24576 17/30 Write block 5 len 24576 data: 6 6 5 5 4 6 18/30 Read block 72 len 20480 19/30 Read block 376 len 4096 20/30 Write block 235 len 8192 data: 5 4 21/30 Read block 85 len 12288 22/30 Read block 364 len 20480 23/30 Read block 284 len 40960 24/30 Read block 372 len 36864 25/30 Read block 92 len 8192 26/30 Write block 514 len 12288 data: 5 6 6 27/30 Write block 290 len 36864 data: 7 7 5 8 10 9 8 8 8 28/30 Write block 51 len 24576 data: 8 7 6 6 6 7 29/30 Read block 40 len 36864 30/30 Read block 96 len 40960 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 8 Done Done Sent false 1002 Acked Read 1 Done Done Sent false 1003 Acked Write 1 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:23.538350445Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:23.538530144Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1373,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 109 109 109 T T F <--- 1 020-039 58 58 58 114 114 114 F F F 2 040-059 58 58 58 114 114 114 T T F <--- 3 060-079 58 58 58 114 114 114 F F F 4 080-099 60 60 54 115 115 106 F F F <--- 5 100-119 56 56 56 112 112 112 F F F 6 120-139 56 56 56 112 112 112 T T F <--- 7 140-159 58 58 58 114 114 114 F F F 8 160-179 58 58 58 114 114 114 F F F 9 180-199 58 58 58 114 114 114 F F F 10 200-219 57 57 57 113 113 113 F F F 11 220-239 60 60 58 116 116 114 T T F <--- 12 240-259 58 58 58 114 114 114 F F F 13 260-279 55 55 55 107 107 107 F F F 14 280-299 58 58 58 114 114 114 T T F <--- 15 300-319 57 57 57 113 113 113 F F F 16 320-339 58 58 58 114 114 114 F F F 17 340-359 52 52 52 100 100 100 F F F 18 360-379 58 58 58 114 114 114 F F F 19 380-399 60 60 58 115 115 114 F F T <--- 20 400-419 60 60 58 116 116 114 F F T <--- 21 420-439 60 60 58 116 116 114 F F F <--- 22 440-459 55 55 55 107 107 107 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 58 58 58 114 114 114 F F F 25 500-519 53 53 53 103 103 103 T T F <--- 26 520-539 56 56 56 108 108 108 F F F 27 540-559 54 54 54 104 104 104 F F F 28 560-579 58 58 58 114 114 114 F F F 29 580-599 58 58 58 114 114 114 F F F Max gen: 60, Max flush: 116 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 117 117 117 F F F 1 020-039 58 58 58 114 114 114 F F F 2 040-059 61 61 61 117 117 117 F F F 3 060-079 58 58 58 114 114 114 F F F 4 080-099 60 60 60 115 115 115 F F F 5 100-119 56 56 56 112 112 112 F F F 6 120-139 61 61 61 117 117 117 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 58 58 58 114 114 114 F F F 9 180-199 58 58 58 114 114 114 F F F 10 200-219 57 57 57 113 113 113 F F F 11 220-239 61 61 61 117 117 117 F F F 12 240-259 58 58 58 114 114 114 F F F 13 260-279 55 55 55 107 107 107 F F F 14 280-299 61 61 61 117 117 117 F F F 15 300-319 57 57 57 113 113 113 F F F 16 320-339 58 58 58 114 114 114 F F F 17 340-359 52 52 52 100 100 100 F F F 18 360-379 58 58 58 114 114 114 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 60 60 60 116 116 116 F F F 21 420-439 60 60 60 116 116 116 F F F 22 440-459 55 55 55 107 107 107 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 58 58 58 114 114 114 F F F 25 500-519 61 61 61 117 117 117 F F F 26 520-539 56 56 56 108 108 108 F F F 27 540-559 54 54 54 104 104 104 F F F 28 560-579 58 58 58 114 114 114 F F F 29 580-599 58 58 58 114 114 114 F F F Max gen: 61, Max flush: 117 Begin loop 31 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.779606368Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.787674606Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.787946964Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.7879817Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.787991409Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788012817Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381} {"msg":"Crucible 2bfc59dc-8403-4ef4-8e26-ff0f7f1b9389 has session id: 3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788292494Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"Upstairs opts: Upstairs UUID: 2bfc59dc-8403-4ef4-8e26-ff0f7f1b9389, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788333049Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"Crucible stats registered with UUID: 2bfc59dc-8403-4ef4-8e26-ff0f7f1b9389","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788343458Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:27.788354477Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788476573Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788492201Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.78850214Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788647063Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.7886697Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381} {"msg":"2bfc59dc-8403-4ef4-8e26-ff0f7f1b9389 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788699566Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788708935Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"client":"0","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788718634Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"client":"1","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788728323Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"client":"2","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788737772Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788747821Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.7887577Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"ds_connection connected from Ok(127.0.0.1:52340)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788829341Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48096)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788851199Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"ds_connection connected from Ok(127.0.0.1:45966)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.788862317Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789552437Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"client":"1","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789574794Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"client":"2","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789604141Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"client":"0","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789862521Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789875209Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"[0]R flush_numbers[0..12]: [117, 114, 117, 114, 115, 112, 117, 114, 114, 114, 113, 117]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789891407Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"downstairs","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"[0]R generation[0..12]: [61, 58, 61, 58, 60, 56, 61, 58, 58, 58, 57, 61]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789902026Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"downstairs","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789913135Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"downstairs","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"[1]R flush_numbers[0..12]: [117, 114, 117, 114, 115, 112, 117, 114, 114, 114, 113, 117]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789923734Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"downstairs","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"[1]R generation[0..12]: [61, 58, 61, 58, 60, 56, 61, 58, 58, 58, 57, 61]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789934032Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"downstairs","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789944901Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"downstairs","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"[2]R flush_numbers[0..12]: [117, 114, 117, 114, 115, 112, 117, 114, 114, 114, 113, 117]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.78995548Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"downstairs","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"[2]R generation[0..12]: [61, 58, 61, 58, 60, 56, 61, 58, 58, 58, 57, 61]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789965919Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"downstairs","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789976807Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"downstairs","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.789988906Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"downstairs","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.790012073Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"downstairs","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.790021552Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"downstairs","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.790030921Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"":"downstairs","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.7900408Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.790049719Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"client":"0","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.790059228Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"client":"1","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.790068817Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"client":"2","":"downstairs","session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.790078536Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.790087654Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"2bfc59dc-8403-4ef4-8e26-ff0f7f1b9389 is now active with session: 3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.790097583Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.790106262Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"session_id":"3fe7e8ae-1b1a-4e7d-8c15-aad23c080be8"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.790115441Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381} {"msg":"Activated sub_volume 2bfc59dc-8403-4ef4-8e26-ff0f7f1b9389","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.79012342Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.790131429Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.790410027Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"task":"crutest"} 01/30 Write block 359 len 28672 data: 5 6 6 5 5 5 5 02/30 Flush 03/30 Write block 512 len 32768 data: 4 5 6 7 7 6 6 6 04/30 Write block 536 len 12288 data: 6 4 4 05/30 Write block 411 len 40960 data: 4 4 3 4 3 4 3 4 4 6 06/30 Write block 439 len 28672 data: 4 5 7 7 8 8 8 07/30 Write block 571 len 16384 data: 7 8 7 8 08/30 Read block 593 len 8192 09/30 Write block 71 len 24576 data: 7 7 8 7 5 4 10/30 Read block 4 len 20480 11/30 Read block 357 len 16384 12/30 Read block 28 len 36864 13/30 Read block 182 len 12288 14/30 Read block 466 len 16384 15/30 Write block 235 len 16384 data: 6 5 6 8 16/30 Read block 534 len 32768 17/30 Read block 322 len 32768 18/30 Read block 185 len 28672 19/30 Read block 304 len 24576 20/30 Write block 125 len 32768 data: 8 9 8 8 8 8 6 5 21/30 Read block 485 len 28672 22/30 Flush 23/30 Write block 438 len 20480 data: 3 5 6 8 8 24/30 Read block 76 len 16384 25/30 Flush 26/30 Write block 536 len 12288 data: 7 5 5 27/30 Read block 343 len 36864 28/30 Write block 93 len 24576 data: 7 7 7 6 4 6 29/30 Write block 291 len 16384 data: 8 6 9 11 30/30 Write block 122 len 20480 data: 5 7 8 9 10 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 8 Done Sent Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 5 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 28 3 34 Done 25 0 25 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1001)) Some(JobId(1024)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:27.798929132Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:27.799091943Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1381,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 117 117 117 F F F 1 020-039 58 58 58 114 114 114 F F F 2 040-059 61 61 61 117 117 117 F F F 3 060-079 62 58 62 119 114 119 F F F <--- 4 080-099 60 60 60 115 115 115 T F T <--- 5 100-119 56 56 56 112 112 112 F F F 6 120-139 62 61 62 119 117 119 T F T <--- 7 140-159 58 58 58 114 114 114 F F F 8 160-179 58 58 58 114 114 114 F F F 9 180-199 58 58 58 114 114 114 F F F 10 200-219 57 57 57 113 113 113 F F F 11 220-239 62 61 62 119 117 119 F F F <--- 12 240-259 58 58 58 114 114 114 F F F 13 260-279 55 55 55 107 107 107 F F F 14 280-299 61 61 61 117 117 117 T F T <--- 15 300-319 57 57 57 113 113 113 F F F 16 320-339 58 58 58 114 114 114 F F F 17 340-359 62 62 62 118 118 118 F F F 18 360-379 62 62 62 118 118 118 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 62 60 62 119 116 119 F F F <--- 21 420-439 62 60 62 120 116 120 F F F <--- 22 440-459 62 55 62 120 107 120 F F F <--- 23 460-479 45 45 45 82 82 82 F F F 24 480-499 58 58 58 114 114 114 F F F 25 500-519 62 61 62 119 117 119 F F F <--- 26 520-539 62 56 62 119 108 119 T F T <--- 27 540-559 54 54 54 104 104 104 F F F 28 560-579 62 58 62 119 114 119 F F F <--- 29 580-599 58 58 58 114 114 114 F F F Max gen: 62, Max flush: 120 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 61 61 61 117 117 117 F F F 1 020-039 58 58 58 114 114 114 F F F 2 040-059 61 61 61 117 117 117 F F F 3 060-079 62 62 62 119 119 119 F F F 4 080-099 63 63 63 121 121 121 F F F 5 100-119 56 56 56 112 112 112 F F F 6 120-139 63 63 63 121 121 121 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 58 58 58 114 114 114 F F F 9 180-199 58 58 58 114 114 114 F F F 10 200-219 57 57 57 113 113 113 F F F 11 220-239 62 62 62 119 119 119 F F F 12 240-259 58 58 58 114 114 114 F F F 13 260-279 55 55 55 107 107 107 F F F 14 280-299 63 63 63 121 121 121 F F F 15 300-319 57 57 57 113 113 113 F F F 16 320-339 58 58 58 114 114 114 F F F 17 340-359 62 62 62 118 118 118 F F F 18 360-379 62 62 62 118 118 118 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 62 62 62 119 119 119 F F F 21 420-439 62 62 62 120 120 120 F F F 22 440-459 62 62 62 120 120 120 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 58 58 58 114 114 114 F F F 25 500-519 62 62 62 119 119 119 F F F 26 520-539 63 63 63 121 121 121 F F F 27 540-559 54 54 54 104 104 104 F F F 28 560-579 62 62 62 119 119 119 F F F 29 580-599 58 58 58 114 114 114 F F F Max gen: 63, Max flush: 121 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.069406873Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.077610826Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.077881284Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.077910971Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.07791927Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.077931088Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389} {"msg":"Crucible 28208fb9-f82f-41ea-b04b-76cb6ebc2311 has session id: 1311d6fa-f2d8-4434-8cf4-32be3088225b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078332991Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"Upstairs opts: Upstairs UUID: 28208fb9-f82f-41ea-b04b-76cb6ebc2311, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078366828Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"Crucible stats registered with UUID: 28208fb9-f82f-41ea-b04b-76cb6ebc2311","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078377506Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:32.078386675Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078537908Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078568094Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078579173Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078692659Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078708588Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389} {"msg":"28208fb9-f82f-41ea-b04b-76cb6ebc2311 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078727335Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078736474Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"client":"0","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078746133Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"client":"1","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078755672Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"client":"2","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078765161Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.07877513Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078785429Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44695)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078849441Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41998)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078866019Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37838)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.078876418Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.079556309Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"client":"0","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.079585155Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"client":"1","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.079623731Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"client":"2","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.079841675Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.079875131Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"[0]R flush_numbers[0..12]: [117, 114, 117, 119, 121, 112, 121, 114, 114, 114, 113, 119]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.079922966Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"downstairs","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"[0]R generation[0..12]: [61, 58, 61, 62, 63, 56, 63, 58, 58, 58, 57, 62]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.079936684Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"downstairs","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.079947653Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"downstairs","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"[1]R flush_numbers[0..12]: [117, 114, 117, 119, 121, 112, 121, 114, 114, 114, 113, 119]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.079958172Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"downstairs","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"[1]R generation[0..12]: [61, 58, 61, 62, 63, 56, 63, 58, 58, 58, 57, 62]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.079968501Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"downstairs","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.079979509Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"downstairs","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"[2]R flush_numbers[0..12]: [117, 114, 117, 119, 121, 112, 121, 114, 114, 114, 113, 119]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.079990108Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"downstairs","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"[2]R generation[0..12]: [61, 58, 61, 62, 63, 56, 63, 58, 58, 58, 57, 62]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080000617Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"downstairs","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080011716Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"downstairs","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080021084Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"downstairs","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080030883Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"downstairs","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"Next flush: 122","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080040322Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"downstairs","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080049721Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"":"downstairs","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.08005953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080068469Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"client":"0","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080079628Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"client":"1","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080089516Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"client":"2","":"downstairs","session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080099595Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080108994Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"28208fb9-f82f-41ea-b04b-76cb6ebc2311 is now active with session: 1311d6fa-f2d8-4434-8cf4-32be3088225b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080118833Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080127582Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"session_id":"1311d6fa-f2d8-4434-8cf4-32be3088225b"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.08014197Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389} {"msg":"Activated sub_volume 28208fb9-f82f-41ea-b04b-76cb6ebc2311","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080150039Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080158188Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.080450784Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"task":"crutest"} 01/30 Write block 595 len 8192 data: 3 4 02/30 Write block 165 len 36864 data: 6 6 6 7 6 4 5 6 6 03/30 Write block 4 len 40960 data: 6 7 7 6 6 5 7 6 7 8 04/30 Write block 298 len 4096 data: 9 05/30 Read block 323 len 24576 06/30 Read block 234 len 24576 07/30 Write block 247 len 8192 data: 5 6 08/30 Read block 223 len 8192 09/30 Write block 85 len 40960 data: 6 8 9 9 10 9 7 8 8 8 10/30 Read block 283 len 32768 11/30 Read block 204 len 20480 12/30 Read block 362 len 20480 13/30 Read block 225 len 32768 14/30 Write block 82 len 40960 data: 6 7 6 7 9 10 10 11 10 8 15/30 Write block 213 len 28672 data: 2 2 2 2 3 4 3 16/30 Read block 90 len 8192 17/30 Write block 581 len 8192 data: 7 7 18/30 Write block 30 len 32768 data: 13 15 15 13 10 9 7 4 19/30 Read block 283 len 36864 20/30 Write block 558 len 28672 data: 4 4 3 5 5 5 7 21/30 Write block 441 len 8192 data: 9 9 22/30 Read block 182 len 24576 23/30 Write block 351 len 12288 data: 5 7 7 24/30 Flush 25/30 Write block 489 len 32768 data: 3 6 5 9 9 8 8 7 26/30 Write block 22 len 4096 data: 5 27/30 Read block 102 len 32768 28/30 Read block 390 len 36864 29/30 Write block 268 len 16384 data: 7 8 9 7 30/30 Read block 339 len 8192 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Write 9 Done Done Sent false 1002 Acked Write 10 Done Done Sent false 1003 Acked Write 1 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Write 2 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 8 Done Done Sent false 1027 Acked Read 9 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:32.089239208Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:32.089401489Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1389,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 61 122 122 117 F F F <--- 1 020-039 64 64 58 122 122 114 T T F <--- 2 040-059 61 61 61 117 117 117 F F F 3 060-079 62 62 62 119 119 119 F F F 4 080-099 64 64 63 122 122 121 F F F <--- 5 100-119 56 56 56 112 112 112 F F F 6 120-139 63 63 63 121 121 121 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 64 64 58 122 122 114 F F F <--- 9 180-199 58 58 58 114 114 114 F F F 10 200-219 64 64 57 122 122 113 F F F <--- 11 220-239 62 62 62 119 119 119 F F F 12 240-259 64 64 58 122 122 114 F F F <--- 13 260-279 55 55 55 107 107 107 T T F <--- 14 280-299 64 64 63 122 122 121 F F F <--- 15 300-319 57 57 57 113 113 113 F F F 16 320-339 58 58 58 114 114 114 F F F 17 340-359 64 64 62 122 122 118 F F F <--- 18 360-379 62 62 62 118 118 118 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 62 62 62 119 119 119 F F F 21 420-439 62 62 62 120 120 120 F F F 22 440-459 64 64 62 122 122 120 F F F <--- 23 460-479 45 45 45 82 82 82 F F F 24 480-499 58 58 58 114 114 114 T T F <--- 25 500-519 62 62 62 119 119 119 F F F 26 520-539 63 63 63 121 121 121 F F F 27 540-559 64 64 54 122 122 104 F F F <--- 28 560-579 64 64 62 122 122 119 F F F <--- 29 580-599 64 64 58 122 122 114 F F F <--- Max gen: 64, Max flush: 122 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 122 122 122 F F F 1 020-039 65 65 65 123 123 123 F F F 2 040-059 61 61 61 117 117 117 F F F 3 060-079 62 62 62 119 119 119 F F F 4 080-099 64 64 64 122 122 122 F F F 5 100-119 56 56 56 112 112 112 F F F 6 120-139 63 63 63 121 121 121 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 64 64 64 122 122 122 F F F 9 180-199 58 58 58 114 114 114 F F F 10 200-219 64 64 64 122 122 122 F F F 11 220-239 62 62 62 119 119 119 F F F 12 240-259 64 64 64 122 122 122 F F F 13 260-279 65 65 65 123 123 123 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 57 57 57 113 113 113 F F F 16 320-339 58 58 58 114 114 114 F F F 17 340-359 64 64 64 122 122 122 F F F 18 360-379 62 62 62 118 118 118 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 62 62 62 119 119 119 F F F 21 420-439 62 62 62 120 120 120 F F F 22 440-459 64 64 64 122 122 122 F F F 23 460-479 45 45 45 82 82 82 F F F 24 480-499 65 65 65 123 123 123 F F F 25 500-519 62 62 62 119 119 119 F F F 26 520-539 63 63 63 121 121 121 F F F 27 540-559 64 64 64 122 122 122 F F F 28 560-579 64 64 64 122 122 122 F F F 29 580-599 64 64 64 122 122 122 F F F Max gen: 65, Max flush: 123 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.33949602Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:36.346712808Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.346761642Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.353730859Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.353983229Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354009766Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354020585Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354032084Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397} {"msg":"Crucible 2ab87cda-a5d0-4a2f-8e4a-d9b2ac563874 has session id: 41949c4d-3be6-4737-b479-2054306e9ce7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354353096Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"Upstairs opts: Upstairs UUID: 2ab87cda-a5d0-4a2f-8e4a-d9b2ac563874, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354371274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"Crucible stats registered with UUID: 2ab87cda-a5d0-4a2f-8e4a-d9b2ac563874","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354381303Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:36.354390412Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354533355Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354548253Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354558282Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354722983Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354734701Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397} {"msg":"2ab87cda-a5d0-4a2f-8e4a-d9b2ac563874 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354752699Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354761448Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"client":"0","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354771097Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"client":"1","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354781036Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"client":"2","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354792455Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354802334Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354812252Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"ds_connection connected from Ok(127.0.0.1:57777)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354858647Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"ds_connection connected from Ok(127.0.0.1:48122)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354873675Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"ds_connection connected from Ok(127.0.0.1:44388)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.354983912Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.35560333Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"client":"2","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.35569228Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"client":"1","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.355788069Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"client":"0","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.355935901Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.355952149Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"[0]R flush_numbers[0..12]: [122, 123, 117, 119, 122, 112, 121, 114, 122, 114, 122, 119]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356067846Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"downstairs","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"[0]R generation[0..12]: [64, 65, 61, 62, 64, 56, 63, 58, 64, 58, 64, 62]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356079165Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"downstairs","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356090283Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"downstairs","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"[1]R flush_numbers[0..12]: [122, 123, 117, 119, 122, 112, 121, 114, 122, 114, 122, 119]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356103572Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"downstairs","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"[1]R generation[0..12]: [64, 65, 61, 62, 64, 56, 63, 58, 64, 58, 64, 62]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.35611392Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"downstairs","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356124819Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"downstairs","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"[2]R flush_numbers[0..12]: [122, 123, 117, 119, 122, 112, 121, 114, 122, 114, 122, 119]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356135418Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"downstairs","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"[2]R generation[0..12]: [64, 65, 61, 62, 64, 56, 63, 58, 64, 58, 64, 62]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356145637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"downstairs","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356156506Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"downstairs","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356165824Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"downstairs","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356175413Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"downstairs","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"Next flush: 124","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356184632Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"downstairs","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356194001Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"":"downstairs","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.35620366Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356212539Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"client":"0","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356222358Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"client":"1","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356232137Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"client":"2","":"downstairs","session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356242246Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356252844Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"2ab87cda-a5d0-4a2f-8e4a-d9b2ac563874 is now active with session: 41949c4d-3be6-4737-b479-2054306e9ce7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356262833Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356271442Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"session_id":"41949c4d-3be6-4737-b479-2054306e9ce7"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356280561Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397} {"msg":"Activated sub_volume 2ab87cda-a5d0-4a2f-8e4a-d9b2ac563874","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.35628849Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356296669Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.356595404Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"task":"crutest"} 01/30 Write block 215 len 40960 data: 3 3 4 5 4 4 4 4 5 5 02/30 Read block 384 len 20480 03/30 Read block 302 len 12288 04/30 Write block 484 len 32768 data: 4 4 4 3 4 4 7 6 05/30 Read block 496 len 4096 06/30 Read block 286 len 12288 07/30 Read block 212 len 28672 08/30 Read block 87 len 32768 09/30 Flush 10/30 Write block 238 len 40960 data: 9 9 7 7 8 8 9 6 7 6 11/30 Read block 240 len 24576 12/30 Read block 19 len 12288 13/30 Read block 114 len 24576 14/30 Flush 15/30 Write block 493 len 16384 data: 10 9 9 8 16/30 Read block 121 len 40960 17/30 Write block 474 len 8192 data: 6 5 18/30 Read block 241 len 24576 19/30 Read block 191 len 12288 20/30 Write block 349 len 28672 data: 5 5 6 8 8 9 6 21/30 Write block 415 len 8192 data: 4 5 22/30 Read block 418 len 40960 23/30 Write block 412 len 8192 data: 5 4 24/30 Write block 171 len 16384 data: 6 7 7 5 25/30 Write block 20 len 4096 data: 5 26/30 Write block 262 len 8192 data: 5 6 27/30 Write block 75 len 16384 data: 6 5 3 5 28/30 Write block 241 len 12288 data: 8 9 9 29/30 Write block 513 len 24576 data: 6 7 8 8 7 7 30/30 Write block 546 len 28672 data: 9 10 9 11 11 9 9 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 5 Sent Done Done false 1002 Acked Read 3 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Write 6 Sent Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 2 32 Done 1 29 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:36.364931451Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:36.36511729Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1397,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 122 122 122 F F F 1 020-039 65 65 65 123 123 123 F T T <--- 2 040-059 61 61 61 117 117 117 F F F 3 060-079 62 62 62 119 119 119 F T T <--- 4 080-099 64 64 64 122 122 122 F F F 5 100-119 56 56 56 112 112 112 F F F 6 120-139 63 63 63 121 121 121 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 64 64 64 122 122 122 F T T <--- 9 180-199 58 58 58 114 114 114 F F F 10 200-219 64 66 66 122 124 124 T F F <--- 11 220-239 62 66 66 119 125 125 T F F <--- 12 240-259 64 66 66 122 125 125 F T T <--- 13 260-279 65 65 65 123 123 123 F T T <--- 14 280-299 64 64 64 122 122 122 F F F 15 300-319 57 57 57 113 113 113 F F F 16 320-339 58 58 58 114 114 114 F F F 17 340-359 64 64 64 122 122 122 F T T <--- 18 360-379 62 62 62 118 118 118 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 62 62 62 119 119 119 F T T <--- 21 420-439 62 62 62 120 120 120 F F F 22 440-459 64 64 64 122 122 122 F F F 23 460-479 45 45 45 82 82 82 F T T <--- 24 480-499 65 66 66 123 124 124 F T T <--- 25 500-519 62 62 62 119 119 119 F T T <--- 26 520-539 63 63 63 121 121 121 F F F 27 540-559 64 64 64 122 122 122 F T T <--- 28 560-579 64 64 64 122 122 122 F F F 29 580-599 64 64 64 122 122 122 F F F Max gen: 66, Max flush: 125 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 122 122 122 F F F 1 020-039 67 67 67 126 126 126 F F F 2 040-059 61 61 61 117 117 117 F F F 3 060-079 67 67 67 126 126 126 F F F 4 080-099 64 64 64 122 122 122 F F F 5 100-119 56 56 56 112 112 112 F F F 6 120-139 63 63 63 121 121 121 F F F 7 140-159 58 58 58 114 114 114 F F F 8 160-179 67 67 67 126 126 126 F F F 9 180-199 58 58 58 114 114 114 F F F 10 200-219 66 66 66 124 124 124 F F F 11 220-239 66 66 66 125 125 125 F F F 12 240-259 67 67 67 126 126 126 F F F 13 260-279 67 67 67 126 126 126 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 57 57 57 113 113 113 F F F 16 320-339 58 58 58 114 114 114 F F F 17 340-359 67 67 67 126 126 126 F F F 18 360-379 62 62 62 118 118 118 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 67 67 67 126 126 126 F F F 21 420-439 62 62 62 120 120 120 F F F 22 440-459 64 64 64 122 122 122 F F F 23 460-479 67 67 67 126 126 126 F F F 24 480-499 67 67 67 126 126 126 F F F 25 500-519 67 67 67 126 126 126 F F F 26 520-539 63 63 63 121 121 121 F F F 27 540-559 67 67 67 126 126 126 F F F 28 560-579 64 64 64 122 122 122 F F F 29 580-599 64 64 64 122 122 122 F F F Max gen: 67, Max flush: 126 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.622748791Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.631029404Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.631327369Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.631354376Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.631362985Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.631374294Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405} {"msg":"Crucible 358c17a5-b31c-4004-9dc5-1e6c34161402 has session id: 35390985-3223-4d32-88f9-f9ab85755c9d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.631685867Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"Upstairs opts: Upstairs UUID: 358c17a5-b31c-4004-9dc5-1e6c34161402, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.631703955Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"Crucible stats registered with UUID: 358c17a5-b31c-4004-9dc5-1e6c34161402","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.631715604Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:40.631724733Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.631874595Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.631903432Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.631914631Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.632036207Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.632050985Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405} {"msg":"358c17a5-b31c-4004-9dc5-1e6c34161402 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.632068063Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.632076952Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"client":"0","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.632086631Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"client":"1","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.63209626Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"client":"2","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.632106018Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.632116087Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.632126016Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43017)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.632183009Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:55693)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.632208206Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54498)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.632222405Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.632838543Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"client":"1","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.63286688Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"client":"0","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.632964108Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"client":"2","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633144437Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633163605Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"[0]R flush_numbers[0..12]: [122, 126, 117, 126, 122, 112, 121, 114, 126, 114, 124, 125]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633237206Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"downstairs","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"[0]R generation[0..12]: [64, 67, 61, 67, 64, 56, 63, 58, 67, 58, 66, 66]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633248385Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"downstairs","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633259334Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"downstairs","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"[1]R flush_numbers[0..12]: [122, 126, 117, 126, 122, 112, 121, 114, 126, 114, 124, 125]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633269953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"downstairs","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"[1]R generation[0..12]: [64, 67, 61, 67, 64, 56, 63, 58, 67, 58, 66, 66]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633280411Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"downstairs","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.63329146Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"downstairs","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"[2]R flush_numbers[0..12]: [122, 126, 117, 126, 122, 112, 121, 114, 126, 114, 124, 125]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633302019Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"downstairs","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"[2]R generation[0..12]: [64, 67, 61, 67, 64, 56, 63, 58, 67, 58, 66, 66]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633312308Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"downstairs","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633325186Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"downstairs","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633334565Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"downstairs","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633344204Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"downstairs","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633353453Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"downstairs","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633362742Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"":"downstairs","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633372441Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.63338127Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"client":"0","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633390888Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"client":"1","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633400387Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"client":"2","":"downstairs","session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633410226Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633419385Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"358c17a5-b31c-4004-9dc5-1e6c34161402 is now active with session: 35390985-3223-4d32-88f9-f9ab85755c9d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633429274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633437913Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"session_id":"35390985-3223-4d32-88f9-f9ab85755c9d"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633447012Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405} {"msg":"Activated sub_volume 358c17a5-b31c-4004-9dc5-1e6c34161402","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633454991Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.63346302Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.633730889Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"task":"crutest"} 01/30 Write block 24 len 8192 data: 6 6 02/30 Write block 347 len 12288 data: 3 4 6 03/30 Write block 208 len 8192 data: 6 7 04/30 Write block 227 len 36864 data: 6 5 5 4 4 5 5 4 7 05/30 Read block 160 len 12288 06/30 Write block 300 len 4096 data: 9 07/30 Flush 08/30 Write block 336 len 20480 data: 6 6 5 6 6 09/30 Write block 80 len 28672 data: 5 5 7 8 7 8 10 10/30 Flush 11/30 Write block 193 len 36864 data: 6 6 6 6 7 8 5 6 6 12/30 Write block 149 len 36864 data: 5 8 7 8 8 7 6 5 5 13/30 Read block 397 len 20480 14/30 Read block 482 len 16384 15/30 Write block 106 len 20480 data: 5 4 4 2 2 16/30 Flush 17/30 Write block 241 len 32768 data: 9 10 10 10 7 8 7 7 18/30 Read block 222 len 4096 19/30 Read block 299 len 28672 20/30 Flush 21/30 Write block 120 len 32768 data: 5 5 6 8 9 10 11 9 22/30 Flush 23/30 Write block 83 len 16384 data: 9 8 9 11 24/30 Write block 303 len 20480 data: 9 6 8 7 6 25/30 Write block 77 len 16384 data: 4 6 5 6 26/30 Read block 204 len 12288 27/30 Write block 49 len 32768 data: 4 4 9 8 7 7 7 8 28/30 Write block 553 len 16384 data: 10 7 7 6 29/30 Read block 281 len 8192 30/30 Read block 437 len 12288 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Write 3 Done Done Sent false 1002 Acked Write 2 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Read 3 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Read 4 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:40.641415472Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:40.641579693Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1405,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 122 122 122 F F F 1 020-039 68 68 67 127 127 126 F F F <--- 2 040-059 61 61 61 117 117 117 T T F <--- 3 060-079 67 67 67 126 126 126 T T F <--- 4 080-099 68 68 64 128 128 122 T T F <--- 5 100-119 68 68 56 129 129 112 F F F <--- 6 120-139 68 68 63 131 131 121 F F F <--- 7 140-159 68 68 58 129 129 114 F F F <--- 8 160-179 67 67 67 126 126 126 F F F 9 180-199 68 68 58 129 129 114 F F F <--- 10 200-219 68 68 66 129 129 124 F F F <--- 11 220-239 68 68 66 127 127 125 F F F <--- 12 240-259 68 68 67 130 130 126 F F F <--- 13 260-279 67 67 67 126 126 126 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 68 68 57 127 127 113 T T F <--- 16 320-339 68 68 58 128 128 114 F F F <--- 17 340-359 68 68 67 128 128 126 F F F <--- 18 360-379 62 62 62 118 118 118 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 67 67 67 126 126 126 F F F 21 420-439 62 62 62 120 120 120 F F F 22 440-459 64 64 64 122 122 122 F F F 23 460-479 67 67 67 126 126 126 F F F 24 480-499 67 67 67 126 126 126 F F F 25 500-519 67 67 67 126 126 126 F F F 26 520-539 63 63 63 121 121 121 F F F 27 540-559 67 67 67 126 126 126 T T F <--- 28 560-579 64 64 64 122 122 122 F F F 29 580-599 64 64 64 122 122 122 F F F Max gen: 68, Max flush: 131 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 122 122 122 F F F 1 020-039 68 68 68 127 127 127 F F F 2 040-059 69 69 69 132 132 132 F F F 3 060-079 69 69 69 132 132 132 F F F 4 080-099 69 69 69 132 132 132 F F F 5 100-119 68 68 68 129 129 129 F F F 6 120-139 68 68 68 131 131 131 F F F 7 140-159 68 68 68 129 129 129 F F F 8 160-179 67 67 67 126 126 126 F F F 9 180-199 68 68 68 129 129 129 F F F 10 200-219 68 68 68 129 129 129 F F F 11 220-239 68 68 68 127 127 127 F F F 12 240-259 68 68 68 130 130 130 F F F 13 260-279 67 67 67 126 126 126 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 69 69 69 132 132 132 F F F 16 320-339 68 68 68 128 128 128 F F F 17 340-359 68 68 68 128 128 128 F F F 18 360-379 62 62 62 118 118 118 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 67 67 67 126 126 126 F F F 21 420-439 62 62 62 120 120 120 F F F 22 440-459 64 64 64 122 122 122 F F F 23 460-479 67 67 67 126 126 126 F F F 24 480-499 67 67 67 126 126 126 F F F 25 500-519 67 67 67 126 126 126 F F F 26 520-539 63 63 63 121 121 121 F F F 27 540-559 69 69 69 132 132 132 F F F 28 560-579 64 64 64 122 122 122 F F F 29 580-599 64 64 64 122 122 122 F F F Max gen: 69, Max flush: 132 Begin loop 35 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.892265474Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:44.899443726Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.899498749Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.90625872Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.906510771Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.906540687Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.906549157Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.906560605Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413} {"msg":"Crucible 13609458-4a1a-4ec2-889f-4bc62dc7b0e0 has session id: d971aa22-1498-41a6-9107-c49f15b13e64","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.906875298Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"Upstairs opts: Upstairs UUID: 13609458-4a1a-4ec2-889f-4bc62dc7b0e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.906903955Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"Crucible stats registered with UUID: 13609458-4a1a-4ec2-889f-4bc62dc7b0e0","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.906915134Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:44.906924493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907046218Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907064166Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907074425Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907230147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907248685Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413} {"msg":"13609458-4a1a-4ec2-889f-4bc62dc7b0e0 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907277052Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.90728647Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"client":"0","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907296399Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"client":"1","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907306188Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"client":"2","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907315697Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907334275Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907344194Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"ds_connection connected from Ok(127.0.0.1:35640)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907354453Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"ds_connection connected from Ok(127.0.0.1:52609)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907439213Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"ds_connection connected from Ok(127.0.0.1:62273)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.907501115Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908026214Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"client":"1","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908045002Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"client":"2","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908160918Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"client":"0","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908331648Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908355916Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"[0]R flush_numbers[0..12]: [122, 127, 132, 132, 132, 129, 131, 129, 126, 129, 129, 127]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908438656Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"downstairs","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"[0]R generation[0..12]: [64, 68, 69, 69, 69, 68, 68, 68, 67, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908449825Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"downstairs","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908460703Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"downstairs","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"[1]R flush_numbers[0..12]: [122, 127, 132, 132, 132, 129, 131, 129, 126, 129, 129, 127]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908471462Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"downstairs","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"[1]R generation[0..12]: [64, 68, 69, 69, 69, 68, 68, 68, 67, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908481761Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"downstairs","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.90849261Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"downstairs","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"[2]R flush_numbers[0..12]: [122, 127, 132, 132, 132, 129, 131, 129, 126, 129, 129, 127]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908503358Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"downstairs","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"[2]R generation[0..12]: [64, 68, 69, 69, 69, 68, 68, 68, 67, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908513857Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"downstairs","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908524686Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"downstairs","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908534185Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"downstairs","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908543834Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"downstairs","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908554422Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"downstairs","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908563771Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"":"downstairs","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.90857366Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908582589Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"client":"0","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908592188Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"client":"1","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908601907Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"client":"2","":"downstairs","session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908611726Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908620865Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"13609458-4a1a-4ec2-889f-4bc62dc7b0e0 is now active with session: d971aa22-1498-41a6-9107-c49f15b13e64","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908630654Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908639443Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"session_id":"d971aa22-1498-41a6-9107-c49f15b13e64"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908648621Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413} {"msg":"Activated sub_volume 13609458-4a1a-4ec2-889f-4bc62dc7b0e0","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.90865663Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.90866476Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.908959095Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"task":"crutest"} 01/30 Write block 544 len 12288 data: 9 9 10 02/30 Read block 537 len 12288 03/30 Write block 61 len 40960 data: 4 4 4 5 5 4 4 4 4 6 04/30 Read block 26 len 24576 05/30 Read block 43 len 40960 06/30 Write block 117 len 28672 data: 6 7 7 6 6 7 9 07/30 Write block 576 len 16384 data: 8 9 5 7 08/30 Read block 113 len 32768 09/30 Write block 89 len 36864 data: 12 11 9 9 9 9 8 7 5 10/30 Read block 340 len 32768 11/30 Write block 505 len 12288 data: 3 4 5 12/30 Read block 301 len 20480 13/30 Read block 249 len 36864 14/30 Read block 426 len 24576 15/30 Write block 343 len 12288 data: 5 5 4 16/30 Read block 560 len 4096 17/30 Write block 101 len 8192 data: 4 4 18/30 Read block 140 len 32768 19/30 Write block 380 len 20480 data: 4 5 7 8 6 20/30 Read block 300 len 20480 21/30 Write block 177 len 28672 data: 4 4 4 5 5 4 5 22/30 Read block 195 len 36864 23/30 Write block 351 len 16384 data: 7 9 9 10 24/30 Read block 129 len 8192 25/30 Read block 310 len 32768 26/30 Write block 157 len 16384 data: 6 6 9 8 27/30 Write block 303 len 28672 data: 10 7 9 8 7 6 5 28/30 Write block 491 len 8192 data: 7 10 29/30 Read block 279 len 16384 30/30 Write block 76 len 8192 data: 6 5 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Write 4 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Write 5 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Write 2 Sent Done Done false 1028 Acked Read 4 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 1 1 28 Done 4 29 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:44.917730261Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:44.917891912Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1413,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 122 122 122 F F F 1 020-039 68 68 68 127 127 127 F F F 2 040-059 69 69 69 132 132 132 F F F 3 060-079 69 69 69 132 132 132 T T T 4 080-099 69 69 69 132 132 132 F T T <--- 5 100-119 68 68 68 129 129 129 F T T <--- 6 120-139 68 68 68 131 131 131 F T T <--- 7 140-159 68 68 68 129 129 129 F T T <--- 8 160-179 67 67 67 126 126 126 F T T <--- 9 180-199 68 68 68 129 129 129 F T T <--- 10 200-219 68 68 68 129 129 129 F F F 11 220-239 68 68 68 127 127 127 F F F 12 240-259 68 68 68 130 130 130 F F F 13 260-279 67 67 67 126 126 126 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 69 69 69 132 132 132 F T T <--- 16 320-339 68 68 68 128 128 128 F F F 17 340-359 68 68 68 128 128 128 F T T <--- 18 360-379 62 62 62 118 118 118 F F F 19 380-399 60 60 60 115 115 115 F T T <--- 20 400-419 67 67 67 126 126 126 F F F 21 420-439 62 62 62 120 120 120 F F F 22 440-459 64 64 64 122 122 122 F F F 23 460-479 67 67 67 126 126 126 F F F 24 480-499 67 67 67 126 126 126 F T T <--- 25 500-519 67 67 67 126 126 126 F T T <--- 26 520-539 63 63 63 121 121 121 F F F 27 540-559 69 69 69 132 132 132 T T T 28 560-579 64 64 64 122 122 122 F T T <--- 29 580-599 64 64 64 122 122 122 F F F Max gen: 69, Max flush: 132 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 35 Verifying data now Files /tmp/test_repair/test_repair_verify.data and /tmp/test_repair/test_repair_verify.data.last differ 78a79 > 6, 80d80 < 4, diff found after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 122 122 122 F F F 1 020-039 68 68 68 127 127 127 F F F 2 040-059 69 69 69 132 132 132 F F F 3 060-079 70 70 70 133 133 133 F F F 4 080-099 70 70 70 133 133 133 F F F 5 100-119 70 70 70 133 133 133 F F F 6 120-139 70 70 70 133 133 133 F F F 7 140-159 70 70 70 133 133 133 F F F 8 160-179 70 70 70 133 133 133 F F F 9 180-199 70 70 70 133 133 133 F F F 10 200-219 68 68 68 129 129 129 F F F 11 220-239 68 68 68 127 127 127 F F F 12 240-259 68 68 68 130 130 130 F F F 13 260-279 67 67 67 126 126 126 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 70 70 70 133 133 133 F F F 16 320-339 68 68 68 128 128 128 F F F 17 340-359 70 70 70 133 133 133 F F F 18 360-379 62 62 62 118 118 118 F F F 19 380-399 70 70 70 133 133 133 F F F 20 400-419 67 67 67 126 126 126 F F F 21 420-439 62 62 62 120 120 120 F F F 22 440-459 64 64 64 122 122 122 F F F 23 460-479 67 67 67 126 126 126 F F F 24 480-499 70 70 70 133 133 133 F F F 25 500-519 70 70 70 133 133 133 F F F 26 520-539 63 63 63 121 121 121 F F F 27 540-559 70 70 70 133 133 133 F F F 28 560-579 70 70 70 133 133 133 F F F 29 580-599 64 64 64 122 122 122 F F F Max gen: 70, Max flush: 133 Begin loop 36 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.185207273Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.193247845Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.193521623Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.193551649Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.193560048Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.193571587Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424} {"msg":"Crucible 8ef7c149-8384-400f-929b-fcc35275228b has session id: 73e87f4a-c0dc-4933-8791-ca6302b5397f","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.193872612Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"Upstairs opts: Upstairs UUID: 8ef7c149-8384-400f-929b-fcc35275228b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.193903708Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"Crucible stats registered with UUID: 8ef7c149-8384-400f-929b-fcc35275228b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.193914157Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:49.193923176Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194070999Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194102645Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194113454Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194246868Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194270835Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424} {"msg":"8ef7c149-8384-400f-929b-fcc35275228b active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194279834Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194291013Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"client":"0","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194301012Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"client":"1","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194310611Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"client":"2","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194320369Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194330488Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194340487Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"ds_connection connected from Ok(127.0.0.1:64984)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194417858Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55088)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194443265Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33340)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.194454624Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195079741Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"client":"0","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195101888Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"client":"1","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195117666Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"client":"2","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195398904Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195443308Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"[0]R flush_numbers[0..12]: [122, 127, 132, 133, 133, 133, 133, 133, 133, 133, 129, 127]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195482264Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"downstairs","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"[0]R generation[0..12]: [64, 68, 69, 70, 70, 70, 70, 70, 70, 70, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195495742Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"downstairs","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195506801Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"downstairs","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"[1]R flush_numbers[0..12]: [122, 127, 132, 133, 133, 133, 133, 133, 133, 133, 129, 127]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.19551745Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"downstairs","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"[1]R generation[0..12]: [64, 68, 69, 70, 70, 70, 70, 70, 70, 70, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195527988Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"downstairs","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195539057Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"downstairs","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"[2]R flush_numbers[0..12]: [122, 127, 132, 133, 133, 133, 133, 133, 133, 133, 129, 127]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195549696Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"downstairs","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"[2]R generation[0..12]: [64, 68, 69, 70, 70, 70, 70, 70, 70, 70, 68, 68]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195560135Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"downstairs","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195570853Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"downstairs","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"Max found gen is 71","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195580142Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"downstairs","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"Generation requested: 72 >= found:71","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195589721Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"downstairs","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"Next flush: 134","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.19559893Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"downstairs","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195608349Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"":"downstairs","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195618148Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195626847Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"client":"0","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195636386Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"client":"1","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195647055Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"client":"2","":"downstairs","session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195656953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195666002Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"8ef7c149-8384-400f-929b-fcc35275228b is now active with session: 73e87f4a-c0dc-4933-8791-ca6302b5397f","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195676111Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.19568494Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"session_id":"73e87f4a-c0dc-4933-8791-ca6302b5397f"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195693849Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424} {"msg":"Activated sub_volume 8ef7c149-8384-400f-929b-fcc35275228b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195701748Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.195709717Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.19603065Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"task":"crutest"} 01/30 Write block 374 len 28672 data: 2 5 5 3 3 3 5 02/30 Write block 19 len 4096 data: 4 03/30 Write block 126 len 40960 data: 12 10 9 9 9 7 6 5 4 5 04/30 Read block 582 len 8192 05/30 Write block 40 len 40960 data: 5 5 5 4 6 7 7 7 4 5 06/30 Flush 07/30 Write block 329 len 36864 data: 4 5 4 7 9 7 7 7 7 08/30 Read block 194 len 20480 09/30 Read block 225 len 12288 10/30 Read block 383 len 36864 11/30 Write block 488 len 8192 data: 5 5 12/30 Flush 13/30 Write block 450 len 32768 data: 6 4 2 2 2 3 4 4 14/30 Read block 244 len 20480 15/30 Write block 74 len 28672 data: 8 7 6 5 7 6 7 16/30 Read block 344 len 4096 17/30 Read block 402 len 8192 18/30 Write block 8 len 4096 data: 7 19/30 Read block 425 len 20480 20/30 Flush 21/30 Write block 227 len 40960 data: 7 6 6 5 5 6 6 5 8 6 22/30 Read block 574 len 36864 23/30 Read block 420 len 20480 24/30 Read block 477 len 24576 25/30 Read block 132 len 24576 26/30 Write block 488 len 40960 data: 6 6 8 8 11 11 10 10 9 5 27/30 Write block 460 len 40960 data: 5 4 5 5 5 4 5 5 4 4 28/30 Write block 224 len 36864 data: 6 6 7 8 7 7 6 6 7 29/30 Read block 135 len 16384 30/30 Read block 319 len 36864 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 10 Done Done Done false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Write 2 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 8 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Read 6 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 25 25 Done 30 30 5 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:49.205257933Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:49.205430603Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1424,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 64 136 136 122 F F T <--- 1 020-039 68 68 68 127 127 127 F F F 2 040-059 72 72 69 134 134 132 F F T <--- 3 060-079 72 72 70 136 136 133 F F F <--- 4 080-099 72 72 70 136 136 133 F F F <--- 5 100-119 70 70 70 133 133 133 F F F 6 120-139 72 72 70 134 134 133 F F T <--- 7 140-159 70 70 70 133 133 133 F F F 8 160-179 70 70 70 133 133 133 F F F 9 180-199 70 70 70 133 133 133 F F F 10 200-219 68 68 68 129 129 129 F F F 11 220-239 68 68 68 127 127 127 T T F <--- 12 240-259 68 68 68 130 130 130 F F F 13 260-279 67 67 67 126 126 126 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 70 70 70 133 133 133 F F F 16 320-339 72 72 68 135 135 128 F F F <--- 17 340-359 70 70 70 133 133 133 F F F 18 360-379 72 72 62 134 134 118 F F T <--- 19 380-399 72 72 70 134 134 133 F F T <--- 20 400-419 67 67 67 126 126 126 F F F 21 420-439 62 62 62 120 120 120 F F F 22 440-459 72 72 64 136 136 122 F F F <--- 23 460-479 67 67 67 126 126 126 T T F <--- 24 480-499 72 72 70 135 135 133 T T F <--- 25 500-519 70 70 70 133 133 133 F F F 26 520-539 63 63 63 121 121 121 F F F 27 540-559 70 70 70 133 133 133 F F F 28 560-579 70 70 70 133 133 133 F F F 29 580-599 64 64 64 122 122 122 F F F Max gen: 72, Max flush: 136 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 136 136 136 F F F 1 020-039 68 68 68 127 127 127 F F F 2 040-059 72 72 72 134 134 134 F F F 3 060-079 72 72 72 136 136 136 F F F 4 080-099 72 72 72 136 136 136 F F F 5 100-119 70 70 70 133 133 133 F F F 6 120-139 72 72 72 134 134 134 F F F 7 140-159 70 70 70 133 133 133 F F F 8 160-179 70 70 70 133 133 133 F F F 9 180-199 70 70 70 133 133 133 F F F 10 200-219 68 68 68 129 129 129 F F F 11 220-239 73 73 73 137 137 137 F F F 12 240-259 68 68 68 130 130 130 F F F 13 260-279 67 67 67 126 126 126 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 70 70 70 133 133 133 F F F 16 320-339 72 72 72 135 135 135 F F F 17 340-359 70 70 70 133 133 133 F F F 18 360-379 72 72 72 134 134 134 F F F 19 380-399 72 72 72 134 134 134 F F F 20 400-419 67 67 67 126 126 126 F F F 21 420-439 62 62 62 120 120 120 F F F 22 440-459 72 72 72 136 136 136 F F F 23 460-479 73 73 73 137 137 137 F F F 24 480-499 73 73 73 137 137 137 F F F 25 500-519 70 70 70 133 133 133 F F F 26 520-539 63 63 63 121 121 121 F F F 27 540-559 70 70 70 133 133 133 F F F 28 560-579 70 70 70 133 133 133 F F F 29 580-599 64 64 64 122 122 122 F F F Max gen: 73, Max flush: 137 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.455170225Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.463139215Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.463413973Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.463440989Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.463449448Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.463460807Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432} {"msg":"Crucible 590dcab4-c801-44cb-9e58-c583d8f1ca53 has session id: b9a16c73-07d8-4d39-b751-70fa3a93a01a","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.463751013Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"Upstairs opts: Upstairs UUID: 590dcab4-c801-44cb-9e58-c583d8f1ca53, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.46377994Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"Crucible stats registered with UUID: 590dcab4-c801-44cb-9e58-c583d8f1ca53","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.463790839Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:53.463799857Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.463920663Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.463935662Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.46394582Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464063367Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464078925Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432} {"msg":"590dcab4-c801-44cb-9e58-c583d8f1ca53 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464094953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464103862Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"client":"0","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464113541Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"client":"1","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.4641232Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"client":"2","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464132949Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464143057Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464152856Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63068)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464236127Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63064)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464247445Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"ds_connection connected from Ok(127.0.0.1:47984)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464257904Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464815409Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"client":"2","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464938315Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"client":"0","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.464949903Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"client":"1","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465189305Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465213302Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"[0]R flush_numbers[0..12]: [136, 127, 134, 136, 136, 133, 134, 133, 133, 133, 129, 137]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.46523314Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"downstairs","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"[0]R generation[0..12]: [72, 68, 72, 72, 72, 70, 72, 70, 70, 70, 68, 73]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465244399Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"downstairs","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465255768Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"downstairs","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"[1]R flush_numbers[0..12]: [136, 127, 134, 136, 136, 133, 134, 133, 133, 133, 129, 137]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465266396Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"downstairs","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"[1]R generation[0..12]: [72, 68, 72, 72, 72, 70, 72, 70, 70, 70, 68, 73]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465276835Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"downstairs","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465287764Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"downstairs","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"[2]R flush_numbers[0..12]: [136, 127, 134, 136, 136, 133, 134, 133, 133, 133, 129, 137]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465300932Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"downstairs","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"[2]R generation[0..12]: [72, 68, 72, 72, 72, 70, 72, 70, 70, 70, 68, 73]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465311381Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"downstairs","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.4653222Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"downstairs","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465331939Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"downstairs","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465341618Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"downstairs","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465350986Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"downstairs","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465360375Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"":"downstairs","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465370384Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465379223Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"client":"0","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465389252Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"client":"1","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465399091Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"client":"2","":"downstairs","session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.46540899Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465418169Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"590dcab4-c801-44cb-9e58-c583d8f1ca53 is now active with session: b9a16c73-07d8-4d39-b751-70fa3a93a01a","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465428267Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465437026Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"session_id":"b9a16c73-07d8-4d39-b751-70fa3a93a01a"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465445975Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432} {"msg":"Activated sub_volume 590dcab4-c801-44cb-9e58-c583d8f1ca53","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465455274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465463323Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.465779906Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"task":"crutest"} 01/30 Write block 539 len 20480 data: 5 4 5 6 6 02/30 Write block 191 len 36864 data: 8 7 7 7 7 7 8 9 6 03/30 Write block 444 len 16384 data: 9 9 8 8 04/30 Write block 113 len 36864 data: 4 5 5 6 7 8 8 7 7 05/30 Read block 90 len 36864 06/30 Write block 8 len 24576 data: 8 6 8 7 8 9 07/30 Read block 500 len 16384 08/30 Read block 215 len 4096 09/30 Read block 373 len 20480 10/30 Read block 47 len 20480 11/30 Write block 433 len 28672 data: 9 8 7 7 4 4 6 12/30 Read block 339 len 20480 13/30 Read block 480 len 32768 14/30 Flush 15/30 Write block 25 len 16384 data: 7 10 9 11 16/30 Read block 115 len 32768 17/30 Flush 18/30 Write block 147 len 24576 data: 6 6 6 9 8 9 19/30 Read block 54 len 24576 20/30 Write block 59 len 32768 data: 4 4 5 5 5 6 6 5 21/30 Read block 14 len 20480 22/30 Write block 7 len 24576 data: 7 9 7 9 8 9 23/30 Write block 8 len 8192 data: 10 8 24/30 Write block 324 len 36864 data: 5 4 2 2 5 5 6 5 8 25/30 Read block 519 len 12288 26/30 Write block 159 len 32768 data: 10 9 6 5 6 7 7 7 27/30 Read block 53 len 8192 28/30 Write block 61 len 36864 data: 6 6 6 7 7 6 5 5 5 29/30 Read block 301 len 32768 30/30 Write block 518 len 12288 data: 8 7 6 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Write 6 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 21 1 23 Done 29 9 29 67 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) None Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:53.475065592Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:53.475238852Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1432,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 72 74 138 136 138 T T T <--- 1 020-039 74 68 74 139 127 139 F F F <--- 2 040-059 72 72 72 134 134 134 T F T <--- 3 060-079 72 72 72 136 136 136 T F T <--- 4 080-099 72 72 72 136 136 136 F F F 5 100-119 74 70 74 138 133 138 F T F <--- 6 120-139 74 72 74 138 134 138 F T F <--- 7 140-159 70 70 70 133 133 133 T F T <--- 8 160-179 70 70 70 133 133 133 T F T <--- 9 180-199 74 70 74 138 133 138 F T F <--- 10 200-219 68 68 68 129 129 129 F F F 11 220-239 73 73 73 137 137 137 F F F 12 240-259 68 68 68 130 130 130 F F F 13 260-279 67 67 67 126 126 126 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 70 70 70 133 133 133 F F F 16 320-339 72 72 72 135 135 135 T F T <--- 17 340-359 70 70 70 133 133 133 F F F 18 360-379 72 72 72 134 134 134 F F F 19 380-399 72 72 72 134 134 134 F F F 20 400-419 67 67 67 126 126 126 F F F 21 420-439 74 62 74 138 120 138 F F F <--- 22 440-459 74 72 74 138 136 138 F T F <--- 23 460-479 73 73 73 137 137 137 F F F 24 480-499 73 73 73 137 137 137 F F F 25 500-519 70 70 70 133 133 133 T F T <--- 26 520-539 74 63 74 138 121 138 T T T <--- 27 540-559 74 70 74 138 133 138 F T F <--- 28 560-579 70 70 70 133 133 133 F F F 29 580-599 64 64 64 122 122 122 F F F Max gen: 74, Max flush: 139 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 140 140 140 F F F 1 020-039 74 74 74 139 139 139 F F F 2 040-059 75 75 75 140 140 140 F F F 3 060-079 75 75 75 140 140 140 F F F 4 080-099 72 72 72 136 136 136 F F F 5 100-119 74 74 74 138 138 138 F F F 6 120-139 74 74 74 138 138 138 F F F 7 140-159 75 75 75 140 140 140 F F F 8 160-179 75 75 75 140 140 140 F F F 9 180-199 74 74 74 138 138 138 F F F 10 200-219 68 68 68 129 129 129 F F F 11 220-239 73 73 73 137 137 137 F F F 12 240-259 68 68 68 130 130 130 F F F 13 260-279 67 67 67 126 126 126 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 70 70 70 133 133 133 F F F 16 320-339 75 75 75 140 140 140 F F F 17 340-359 70 70 70 133 133 133 F F F 18 360-379 72 72 72 134 134 134 F F F 19 380-399 72 72 72 134 134 134 F F F 20 400-419 67 67 67 126 126 126 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 74 74 74 138 138 138 F F F 23 460-479 73 73 73 137 137 137 F F F 24 480-499 73 73 73 137 137 137 F F F 25 500-519 75 75 75 140 140 140 F F F 26 520-539 75 75 75 140 140 140 F F F 27 540-559 74 74 74 138 138 138 F F F 28 560-579 70 70 70 133 133 133 F F F 29 580-599 64 64 64 122 122 122 F F F Max gen: 75, Max flush: 140 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.731083662Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:57.738203171Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.738258734Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.74515095Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.745419089Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.745446095Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.745454784Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.745466283Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440} {"msg":"Crucible 283380bb-df90-43bd-81c7-e4d4cde3e5e3 has session id: be8895a4-5a79-4f33-9bbf-a912f097ab44","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.745764508Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"Upstairs opts: Upstairs UUID: 283380bb-df90-43bd-81c7-e4d4cde3e5e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.745787216Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"Crucible stats registered with UUID: 283380bb-df90-43bd-81c7-e4d4cde3e5e3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.745799214Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:57.745823431Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.745965085Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.745996561Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.74600747Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.746127486Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.746141964Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440} {"msg":"283380bb-df90-43bd-81c7-e4d4cde3e5e3 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.746159622Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.746168411Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"client":"0","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.74617807Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"client":"1","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.746187679Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"client":"2","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.746197058Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.746207037Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.746216935Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"ds_connection connected from Ok(127.0.0.1:58172)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.746268099Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"ds_connection connected from Ok(127.0.0.1:60901)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.746281368Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"ds_connection connected from Ok(127.0.0.1:61220)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.746291727Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.746960819Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"client":"2","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.746985696Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"client":"1","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747010053Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"client":"0","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747256824Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747266603Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"[0]R flush_numbers[0..12]: [140, 139, 140, 140, 136, 138, 138, 140, 140, 138, 129, 137]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.74729148Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"downstairs","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"[0]R generation[0..12]: [75, 74, 75, 75, 72, 74, 74, 75, 75, 74, 68, 73]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747302839Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"downstairs","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747313857Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"downstairs","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"[1]R flush_numbers[0..12]: [140, 139, 140, 140, 136, 138, 138, 140, 140, 138, 129, 137]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747324516Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"downstairs","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"[1]R generation[0..12]: [75, 74, 75, 75, 72, 74, 74, 75, 75, 74, 68, 73]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747334875Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"downstairs","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747345714Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"downstairs","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"[2]R flush_numbers[0..12]: [140, 139, 140, 140, 136, 138, 138, 140, 140, 138, 129, 137]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747356272Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"downstairs","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"[2]R generation[0..12]: [75, 74, 75, 75, 72, 74, 74, 75, 75, 74, 68, 73]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747366681Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"downstairs","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.74737775Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"downstairs","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747389619Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"downstairs","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747399397Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"downstairs","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747408696Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"downstairs","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747418115Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"":"downstairs","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747427864Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747436553Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"client":"0","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747446082Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"client":"1","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747455721Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"client":"2","":"downstairs","session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.74746542Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747474459Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"283380bb-df90-43bd-81c7-e4d4cde3e5e3 is now active with session: be8895a4-5a79-4f33-9bbf-a912f097ab44","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747484407Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747493186Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"session_id":"be8895a4-5a79-4f33-9bbf-a912f097ab44"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747502275Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440} {"msg":"Activated sub_volume 283380bb-df90-43bd-81c7-e4d4cde3e5e3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747510384Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747518463Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.747817649Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"task":"crutest"} 01/30 Write block 556 len 24576 data: 7 5 5 5 4 6 02/30 Read block 538 len 28672 03/30 Read block 338 len 24576 04/30 Write block 403 len 32768 data: 4 4 4 3 4 4 4 4 05/30 Read block 537 len 4096 06/30 Write block 245 len 4096 data: 8 07/30 Read block 110 len 20480 08/30 Read block 199 len 4096 09/30 Read block 52 len 32768 10/30 Read block 478 len 32768 11/30 Read block 91 len 4096 12/30 Read block 50 len 40960 13/30 Write block 235 len 32768 data: 9 7 7 10 10 8 10 11 14/30 Write block 516 len 16384 data: 9 8 9 8 15/30 Write block 551 len 4096 data: 10 16/30 Write block 528 len 32768 data: 2 3 5 5 6 7 6 6 17/30 Read block 471 len 40960 18/30 Read block 520 len 16384 19/30 Read block 99 len 12288 20/30 Write block 0 len 32768 data: 3 3 4 5 7 8 8 8 21/30 Write block 75 len 8192 data: 8 7 22/30 Read block 139 len 32768 23/30 Write block 409 len 8192 data: 5 5 24/30 Write block 305 len 12288 data: 10 9 8 25/30 Read block 366 len 28672 26/30 Write block 257 len 32768 data: 4 2 2 3 3 6 7 6 27/30 Read block 446 len 32768 28/30 Write block 4 len 16384 data: 8 9 9 9 29/30 Read block 429 len 40960 30/30 Write block 454 len 24576 data: 3 4 5 5 5 6 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 7 Sent Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 8 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:41:57.75663215Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:41:57.756807779Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1440,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 140 140 140 F T T <--- 1 020-039 74 74 74 139 139 139 F F F 2 040-059 75 75 75 140 140 140 F F F 3 060-079 75 75 75 140 140 140 F T T <--- 4 080-099 72 72 72 136 136 136 F F F 5 100-119 74 74 74 138 138 138 F F F 6 120-139 74 74 74 138 138 138 F F F 7 140-159 75 75 75 140 140 140 F F F 8 160-179 75 75 75 140 140 140 F F F 9 180-199 74 74 74 138 138 138 F F F 10 200-219 68 68 68 129 129 129 F F F 11 220-239 73 73 73 137 137 137 F T T <--- 12 240-259 68 68 68 130 130 130 F T T <--- 13 260-279 67 67 67 126 126 126 F T T <--- 14 280-299 64 64 64 122 122 122 F F F 15 300-319 70 70 70 133 133 133 F T T <--- 16 320-339 75 75 75 140 140 140 F F F 17 340-359 70 70 70 133 133 133 F F F 18 360-379 72 72 72 134 134 134 F F F 19 380-399 72 72 72 134 134 134 F F F 20 400-419 67 67 67 126 126 126 F T T <--- 21 420-439 74 74 74 138 138 138 F F F 22 440-459 74 74 74 138 138 138 F T T <--- 23 460-479 73 73 73 137 137 137 F F F 24 480-499 73 73 73 137 137 137 F F F 25 500-519 75 75 75 140 140 140 F T T <--- 26 520-539 75 75 75 140 140 140 F T T <--- 27 540-559 74 74 74 138 138 138 T T T 28 560-579 70 70 70 133 133 133 T T T 29 580-599 64 64 64 122 122 122 F F F Max gen: 75, Max flush: 140 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 38 Verifying data now Files /tmp/test_repair/test_repair_verify.data and /tmp/test_repair/test_repair_verify.data.last differ 553a554 > 10, 555d555 < 9, diff found after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 141 141 141 F F F 1 020-039 74 74 74 139 139 139 F F F 2 040-059 75 75 75 140 140 140 F F F 3 060-079 76 76 76 141 141 141 F F F 4 080-099 72 72 72 136 136 136 F F F 5 100-119 74 74 74 138 138 138 F F F 6 120-139 74 74 74 138 138 138 F F F 7 140-159 75 75 75 140 140 140 F F F 8 160-179 75 75 75 140 140 140 F F F 9 180-199 74 74 74 138 138 138 F F F 10 200-219 68 68 68 129 129 129 F F F 11 220-239 76 76 76 141 141 141 F F F 12 240-259 76 76 76 141 141 141 F F F 13 260-279 76 76 76 141 141 141 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 76 76 76 141 141 141 F F F 16 320-339 75 75 75 140 140 140 F F F 17 340-359 70 70 70 133 133 133 F F F 18 360-379 72 72 72 134 134 134 F F F 19 380-399 72 72 72 134 134 134 F F F 20 400-419 76 76 76 141 141 141 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 76 76 76 141 141 141 F F F 23 460-479 73 73 73 137 137 137 F F F 24 480-499 73 73 73 137 137 137 F F F 25 500-519 76 76 76 141 141 141 F F F 26 520-539 76 76 76 141 141 141 F F F 27 540-559 76 76 76 141 141 141 F F F 28 560-579 76 76 76 141 141 141 F F F 29 580-599 64 64 64 122 122 122 F F F Max gen: 76, Max flush: 141 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.022414109Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.030492246Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.030771223Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.03079709Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.030805899Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.030817598Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451} {"msg":"Crucible c07e3dda-7572-4ec7-a2fb-68a341e55f4a has session id: 711621a4-15cb-4b8e-a5a0-bc6a85179ef2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031220091Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"Upstairs opts: Upstairs UUID: c07e3dda-7572-4ec7-a2fb-68a341e55f4a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031241478Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"Crucible stats registered with UUID: c07e3dda-7572-4ec7-a2fb-68a341e55f4a","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031251637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:02.031260846Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.03140121Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031416648Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031426787Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031578479Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031594147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451} {"msg":"c07e3dda-7572-4ec7-a2fb-68a341e55f4a active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031629253Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031638872Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"client":"0","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031648581Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"client":"1","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.03165827Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"client":"2","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031667909Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031677967Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031687886Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"ds_connection connected from Ok(127.0.0.1:46773)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031698515Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"io task","client":"2","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"ds_connection connected from Ok(127.0.0.1:61758)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031736071Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"io task","client":"1","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"ds_connection connected from Ok(127.0.0.1:59769)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.031788335Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"io task","client":"0","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032366487Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"client":"0","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032392874Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"client":"1","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032528938Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"client":"2","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032661823Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032705957Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"[0]R flush_numbers[0..12]: [141, 139, 140, 141, 136, 138, 138, 140, 140, 138, 129, 141]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032817374Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"downstairs","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"[0]R generation[0..12]: [76, 74, 75, 76, 72, 74, 74, 75, 75, 74, 68, 76]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032828673Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"downstairs","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032839772Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"downstairs","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"[1]R flush_numbers[0..12]: [141, 139, 140, 141, 136, 138, 138, 140, 140, 138, 129, 141]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032850461Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"downstairs","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"[1]R generation[0..12]: [76, 74, 75, 76, 72, 74, 74, 75, 75, 74, 68, 76]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032860899Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"downstairs","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032871808Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"downstairs","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"[2]R flush_numbers[0..12]: [141, 139, 140, 141, 136, 138, 138, 140, 140, 138, 129, 141]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032882187Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"downstairs","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"[2]R generation[0..12]: [76, 74, 75, 76, 72, 74, 74, 75, 75, 74, 68, 76]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032892546Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"downstairs","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032903434Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"downstairs","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"Max found gen is 77","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032912883Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"downstairs","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"Generation requested: 78 >= found:77","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032922502Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"downstairs","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"Next flush: 142","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032931781Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"downstairs","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.03294111Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"":"downstairs","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032951299Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032961468Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"client":"0","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032971017Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"client":"1","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032980805Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"client":"2","":"downstairs","session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.032990674Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.033009082Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"c07e3dda-7572-4ec7-a2fb-68a341e55f4a is now active with session: 711621a4-15cb-4b8e-a5a0-bc6a85179ef2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.033019091Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.03302794Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"session_id":"711621a4-15cb-4b8e-a5a0-bc6a85179ef2"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.033036959Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451} {"msg":"Activated sub_volume c07e3dda-7572-4ec7-a2fb-68a341e55f4a","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.033044888Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.033053037Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.033337154Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"task":"crutest"} 01/30 Write block 553 len 28672 data: 11 8 8 8 6 6 6 02/30 Write block 203 len 12288 data: 6 7 6 03/30 Read block 475 len 28672 04/30 Write block 138 len 32768 data: 5 4 5 4 5 4 5 6 05/30 Write block 46 len 20480 data: 8 8 5 6 5 06/30 Write block 584 len 24576 data: 6 7 6 6 5 8 07/30 Read block 376 len 4096 08/30 Flush 09/30 Write block 589 len 28672 data: 9 7 5 6 4 3 4 10/30 Read block 423 len 28672 11/30 Write block 562 len 28672 data: 6 6 8 6 6 6 6 12/30 Read block 102 len 16384 13/30 Read block 392 len 8192 14/30 Write block 6 len 4096 data: 10 15/30 Write block 578 len 16384 data: 6 8 7 8 16/30 Read block 495 len 40960 17/30 Write block 445 len 8192 data: 10 9 18/30 Write block 332 len 12288 data: 9 10 8 19/30 Write block 231 len 8192 data: 7 8 20/30 Write block 499 len 12288 data: 5 6 4 21/30 Write block 476 len 36864 data: 7 7 8 8 6 5 4 4 5 22/30 Write block 132 len 20480 data: 7 6 5 6 5 23/30 Read block 393 len 4096 24/30 Flush 25/30 Write block 532 len 40960 data: 7 8 7 7 8 6 6 6 5 6 26/30 Read block 439 len 16384 27/30 Read block 172 len 16384 28/30 Read block 470 len 8192 29/30 Read block 261 len 20480 30/30 Write block 67 len 16384 data: 6 6 6 7 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 7 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Write 1 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 10 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Read 2 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 12 1 14 Done 21 10 21 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1007)) Some(JobId(1023)) Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:02.04133681Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:02.041492512Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1451,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 76 78 143 141 143 F T F <--- 1 020-039 74 74 74 139 139 139 F F F 2 040-059 78 78 78 142 142 142 F F F 3 060-079 76 76 76 141 141 141 T F T <--- 4 080-099 72 72 72 136 136 136 F F F 5 100-119 74 74 74 138 138 138 F F F 6 120-139 78 78 78 143 142 143 F F F <--- 7 140-159 78 78 78 142 142 142 F F F 8 160-179 75 75 75 140 140 140 F F F 9 180-199 74 74 74 138 138 138 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 78 76 78 143 141 143 F F F <--- 12 240-259 76 76 76 141 141 141 F F F 13 260-279 76 76 76 141 141 141 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 76 76 76 141 141 141 F F F 16 320-339 78 75 78 143 140 143 F T F <--- 17 340-359 70 70 70 133 133 133 F F F 18 360-379 72 72 72 134 134 134 F F F 19 380-399 72 72 72 134 134 134 F F F 20 400-419 76 76 76 141 141 141 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 78 76 78 143 141 143 F T F <--- 23 460-479 78 73 78 143 137 143 F F F <--- 24 480-499 78 73 78 143 137 143 F F F <--- 25 500-519 78 76 78 143 141 143 F F F <--- 26 520-539 76 76 76 141 141 141 T F T <--- 27 540-559 78 78 78 142 142 142 T F T <--- 28 560-579 78 76 78 143 141 143 F T F <--- 29 580-599 78 78 78 143 142 143 F T F <--- Max gen: 78, Max flush: 143 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 143 143 143 F F F 1 020-039 74 74 74 139 139 139 F F F 2 040-059 78 78 78 142 142 142 F F F 3 060-079 79 79 79 144 144 144 F F F 4 080-099 72 72 72 136 136 136 F F F 5 100-119 74 74 74 138 138 138 F F F 6 120-139 78 78 78 143 143 143 F F F 7 140-159 78 78 78 142 142 142 F F F 8 160-179 75 75 75 140 140 140 F F F 9 180-199 74 74 74 138 138 138 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 78 78 78 143 143 143 F F F 12 240-259 76 76 76 141 141 141 F F F 13 260-279 76 76 76 141 141 141 F F F 14 280-299 64 64 64 122 122 122 F F F 15 300-319 76 76 76 141 141 141 F F F 16 320-339 78 78 78 143 143 143 F F F 17 340-359 70 70 70 133 133 133 F F F 18 360-379 72 72 72 134 134 134 F F F 19 380-399 72 72 72 134 134 134 F F F 20 400-419 76 76 76 141 141 141 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 78 78 78 143 143 143 F F F 23 460-479 78 78 78 143 143 143 F F F 24 480-499 78 78 78 143 143 143 F F F 25 500-519 78 78 78 143 143 143 F F F 26 520-539 79 79 79 144 144 144 F F F 27 540-559 79 79 79 144 144 144 F F F 28 560-579 78 78 78 143 143 143 F F F 29 580-599 78 78 78 143 143 143 F F F Max gen: 79, Max flush: 144 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.287936409Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.296115254Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.296412579Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.296440796Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.296449325Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.296460974Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459} {"msg":"Crucible eea5ff5e-bfc3-4020-aac9-6eeed2f41701 has session id: 9b5a3def-d6ab-4e3b-ac30-ba514bab7222","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.296786076Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"Upstairs opts: Upstairs UUID: eea5ff5e-bfc3-4020-aac9-6eeed2f41701, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.296804894Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"Crucible stats registered with UUID: eea5ff5e-bfc3-4020-aac9-6eeed2f41701","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.296814762Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:06.296823701Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.296961885Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.296990542Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.297001521Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.297129876Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.297147854Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459} {"msg":"eea5ff5e-bfc3-4020-aac9-6eeed2f41701 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.297164912Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.29717383Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"client":"0","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.297183599Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"client":"1","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.297193208Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"client":"2","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.297202847Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.297212676Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.297224565Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"ds_connection connected from Ok(127.0.0.1:59198)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.297341701Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"io task","client":"0","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"ds_connection connected from Ok(127.0.0.1:61780)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.297451368Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"io task","client":"1","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"ds_connection connected from Ok(127.0.0.1:56516)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.297463287Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"io task","client":"2","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.297992235Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"client":"1","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298012723Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"client":"0","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298023691Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"client":"2","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298304579Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298337065Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"[0]R flush_numbers[0..12]: [143, 139, 142, 144, 136, 138, 143, 142, 140, 138, 142, 143]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.29837649Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"downstairs","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"[0]R generation[0..12]: [78, 74, 78, 79, 72, 74, 78, 78, 75, 74, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298388689Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"downstairs","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298399807Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"downstairs","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"[1]R flush_numbers[0..12]: [143, 139, 142, 144, 136, 138, 143, 142, 140, 138, 142, 143]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298410536Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"downstairs","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"[1]R generation[0..12]: [78, 74, 78, 79, 72, 74, 78, 78, 75, 74, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298420915Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"downstairs","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298434573Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"downstairs","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"[2]R flush_numbers[0..12]: [143, 139, 142, 144, 136, 138, 143, 142, 140, 138, 142, 143]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298445102Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"downstairs","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"[2]R generation[0..12]: [78, 74, 78, 79, 72, 74, 78, 78, 75, 74, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298455231Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"downstairs","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.29846609Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"downstairs","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298475449Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"downstairs","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298485147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"downstairs","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298494416Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"downstairs","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298503825Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"":"downstairs","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298513864Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298522783Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"client":"0","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298532682Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"client":"1","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298542581Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"client":"2","":"downstairs","session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.29855259Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298561708Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"eea5ff5e-bfc3-4020-aac9-6eeed2f41701 is now active with session: 9b5a3def-d6ab-4e3b-ac30-ba514bab7222","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298571587Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298581776Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"session_id":"9b5a3def-d6ab-4e3b-ac30-ba514bab7222"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298590775Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459} {"msg":"Activated sub_volume eea5ff5e-bfc3-4020-aac9-6eeed2f41701","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298598834Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-09-16T04:42:06.298606933Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.298848615Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"task":"crutest"} 01/30 Write block 51 len 4096 data: 10 02/30 Write block 186 len 40960 data: 3 3 4 8 8 9 8 8 8 8 03/30 Read block 411 len 8192 04/30 Read block 169 len 20480 05/30 Read block 387 len 20480 06/30 Read block 138 len 20480 07/30 Write block 488 len 36864 data: 7 7 9 9 12 12 11 11 10 08/30 Flush 09/30 Write block 122 len 4096 data: 8 10/30 Write block 334 len 28672 data: 9 8 8 8 6 7 7 11/30 Read block 140 len 4096 12/30 Write block 506 len 8192 data: 5 6 13/30 Write block 506 len 40960 data: 6 7 5 4 5 5 5 7 8 9 14/30 Read block 473 len 4096 15/30 Flush 16/30 Write block 327 len 36864 data: 3 6 6 7 6 10 11 10 9 17/30 Flush 18/30 Write block 146 len 32768 data: 6 7 7 7 10 9 10 9 19/30 Write block 80 len 4096 data: 8 20/30 Read block 246 len 12288 21/30 Read block 158 len 16384 22/30 Write block 76 len 4096 data: 8 23/30 Flush 24/30 Write block 26 len 16384 data: 11 10 12 13 25/30 Read block 54 len 32768 26/30 Write block 276 len 40960 data: 11 10 11 12 10 8 7 5 5 5 27/30 Write block 190 len 8192 data: 9 10 28/30 Write block 454 len 20480 data: 4 5 6 6 6 29/30 Write block 56 len 24576 data: 9 7 6 5 5 7 30/30 Read block 461 len 24576 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 9 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Write 5 Done Sent Sent false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 24 3 27 Done 30 6 27 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:06.306999914Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:06.307180662Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1459,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 143 143 143 F F F 1 020-039 74 74 74 139 139 139 T F T <--- 2 040-059 80 78 80 145 142 145 T T T <--- 3 060-079 80 79 80 148 144 148 T F T <--- 4 080-099 80 72 80 148 136 148 F F F <--- 5 100-119 74 74 74 138 138 138 F F F 6 120-139 80 78 80 146 143 146 F F F <--- 7 140-159 80 78 80 148 142 148 F F F <--- 8 160-179 75 75 75 140 140 140 F F F 9 180-199 80 74 80 145 138 145 T T T <--- 10 200-219 78 78 78 142 142 142 F F F 11 220-239 78 78 78 143 143 143 F F F 12 240-259 76 76 76 141 141 141 F F F 13 260-279 76 76 76 141 141 141 T F T <--- 14 280-299 64 64 64 122 122 122 T F T <--- 15 300-319 76 76 76 141 141 141 F F F 16 320-339 80 78 80 147 143 147 F F F <--- 17 340-359 80 70 80 146 133 146 F F F <--- 18 360-379 72 72 72 134 134 134 F F F 19 380-399 72 72 72 134 134 134 F F F 20 400-419 76 76 76 141 141 141 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 78 78 78 143 143 143 T F T <--- 23 460-479 78 78 78 143 143 143 F F F 24 480-499 80 78 80 145 143 145 F F F <--- 25 500-519 80 78 80 146 143 146 F F F <--- 26 520-539 79 79 79 144 144 144 F F F 27 540-559 79 79 79 144 144 144 F F F 28 560-579 78 78 78 143 143 143 F F F 29 580-599 78 78 78 143 143 143 F F F Max gen: 80, Max flush: 148 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 143 143 143 F F F 1 020-039 81 81 81 149 149 149 F F F 2 040-059 81 81 81 149 149 149 F F F 3 060-079 81 81 81 149 149 149 F F F 4 080-099 80 80 80 148 148 148 F F F 5 100-119 74 74 74 138 138 138 F F F 6 120-139 80 80 80 146 146 146 F F F 7 140-159 80 80 80 148 148 148 F F F 8 160-179 75 75 75 140 140 140 F F F 9 180-199 81 81 81 149 149 149 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 78 78 78 143 143 143 F F F 12 240-259 76 76 76 141 141 141 F F F 13 260-279 81 81 81 149 149 149 F F F 14 280-299 81 81 81 149 149 149 F F F 15 300-319 76 76 76 141 141 141 F F F 16 320-339 80 80 80 147 147 147 F F F 17 340-359 80 80 80 146 146 146 F F F 18 360-379 72 72 72 134 134 134 F F F 19 380-399 72 72 72 134 134 134 F F F 20 400-419 76 76 76 141 141 141 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 81 81 81 149 149 149 F F F 23 460-479 78 78 78 143 143 143 F F F 24 480-499 80 80 80 145 145 145 F F F 25 500-519 80 80 80 146 146 146 F F F 26 520-539 79 79 79 144 144 144 F F F 27 540-559 79 79 79 144 144 144 F F F 28 560-579 78 78 78 143 143 143 F F F 29 580-599 78 78 78 143 143 143 F F F Max gen: 81, Max flush: 149 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.557961223Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.566071916Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.566343715Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.566368482Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.566377421Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.566388899Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467} {"msg":"Crucible 6574b6c2-0b40-46d3-ae9d-8a877e6957c0 has session id: 7640441b-fb37-4e1d-a0bf-078374d50940","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.566677126Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"Upstairs opts: Upstairs UUID: 6574b6c2-0b40-46d3-ae9d-8a877e6957c0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.566696434Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"Crucible stats registered with UUID: 6574b6c2-0b40-46d3-ae9d-8a877e6957c0","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.566706362Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:10.566715341Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.566893331Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.566922147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.566933216Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567057311Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.5670693Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467} {"msg":"6574b6c2-0b40-46d3-ae9d-8a877e6957c0 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567102446Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567111965Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"client":"0","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567121514Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"client":"1","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567131213Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"client":"2","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567140562Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567150551Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567160459Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"ds_connection connected from Ok(127.0.0.1:39055)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567211163Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"io task","client":"1","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"ds_connection connected from Ok(127.0.0.1:61152)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567222102Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"io task","client":"0","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"ds_connection connected from Ok(127.0.0.1:59249)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567285635Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"io task","client":"2","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567881025Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"client":"2","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567909002Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"client":"0","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.567944838Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"client":"1","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568199308Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568210167Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"[0]R flush_numbers[0..12]: [143, 149, 149, 149, 148, 138, 146, 148, 140, 149, 142, 143]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568243263Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"downstairs","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"[0]R generation[0..12]: [78, 81, 81, 81, 80, 74, 80, 80, 75, 81, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568254762Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"downstairs","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.56826565Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"downstairs","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"[1]R flush_numbers[0..12]: [143, 149, 149, 149, 148, 138, 146, 148, 140, 149, 142, 143]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568276529Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"downstairs","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"[1]R generation[0..12]: [78, 81, 81, 81, 80, 74, 80, 80, 75, 81, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568286918Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"downstairs","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568297727Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"downstairs","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"[2]R flush_numbers[0..12]: [143, 149, 149, 149, 148, 138, 146, 148, 140, 149, 142, 143]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568308475Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"downstairs","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"[2]R generation[0..12]: [78, 81, 81, 81, 80, 74, 80, 80, 75, 81, 78, 78]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568318884Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"downstairs","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568329653Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"downstairs","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568339072Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"downstairs","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.56835031Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"downstairs","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"Next flush: 150","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568359619Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"downstairs","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568369208Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"":"downstairs","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568378957Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568387756Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"client":"0","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568397495Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"client":"1","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568407244Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"client":"2","":"downstairs","session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568416973Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568426082Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"6574b6c2-0b40-46d3-ae9d-8a877e6957c0 is now active with session: 7640441b-fb37-4e1d-a0bf-078374d50940","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.5684361Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568445129Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"session_id":"7640441b-fb37-4e1d-a0bf-078374d50940"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568454158Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467} {"msg":"Activated sub_volume 6574b6c2-0b40-46d3-ae9d-8a877e6957c0","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568462137Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568470186Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.568730146Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"task":"crutest"} 01/30 Write block 222 len 8192 data: 5 6 02/30 Write block 19 len 16384 data: 5 6 4 6 03/30 Read block 332 len 36864 04/30 Read block 119 len 40960 05/30 Write block 472 len 24576 data: 4 6 7 6 8 8 06/30 Write block 155 len 28672 data: 7 6 7 7 11 10 7 07/30 Read block 534 len 36864 08/30 Read block 364 len 4096 09/30 Write block 492 len 24576 data: 13 13 12 12 11 6 10/30 Read block 453 len 20480 11/30 Flush 12/30 Write block 397 len 16384 data: 6 8 9 10 13/30 Flush 14/30 Write block 244 len 36864 data: 11 9 9 8 8 6 8 9 9 15/30 Read block 199 len 28672 16/30 Read block 89 len 24576 17/30 Flush 18/30 Write block 305 len 24576 data: 11 10 9 7 6 4 19/30 Read block 4 len 20480 20/30 Flush 21/30 Write block 178 len 8192 data: 5 5 22/30 Read block 442 len 8192 23/30 Write block 53 len 16384 data: 8 8 8 10 24/30 Read block 55 len 12288 25/30 Flush 26/30 Write block 449 len 8192 data: 5 7 27/30 Write block 501 len 20480 data: 5 3 3 3 4 28/30 Write block 237 len 16384 data: 8 11 11 9 29/30 Write block 119 len 24576 data: 9 8 8 9 10 10 30/30 Read block 81 len 20480 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Sent Done false 1001 Acked Write 4 Done Sent Done false 1002 Acked Read 9 Done Sent Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Write 4 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Write 6 Done Sent Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:10.577110298Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:10.577293387Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1467,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 78 82 150 143 150 F F F <--- 1 020-039 82 81 82 150 149 150 F F F <--- 2 040-059 82 81 82 154 149 154 F F F <--- 3 060-079 81 81 81 149 149 149 F F F 4 080-099 80 80 80 148 148 148 F F F 5 100-119 74 74 74 138 138 138 T F T <--- 6 120-139 80 80 80 146 146 146 T F T <--- 7 140-159 82 80 82 150 148 150 F F F <--- 8 160-179 82 75 82 154 140 154 F F F <--- 9 180-199 81 81 81 149 149 149 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 82 78 82 150 143 150 T F T <--- 12 240-259 82 76 82 152 141 152 T F T <--- 13 260-279 81 81 81 149 149 149 F F F 14 280-299 81 81 81 149 149 149 F F F 15 300-319 82 76 82 153 141 153 F F F <--- 16 320-339 80 80 80 147 147 147 F F F 17 340-359 80 80 80 146 146 146 F F F 18 360-379 72 72 72 134 134 134 F F F 19 380-399 82 72 82 151 134 151 F F F <--- 20 400-419 82 76 82 151 141 151 F F F <--- 21 420-439 74 74 74 138 138 138 F F F 22 440-459 81 81 81 149 149 149 T F T <--- 23 460-479 82 78 82 150 143 150 F F F <--- 24 480-499 82 80 82 150 145 150 F F F <--- 25 500-519 80 80 80 146 146 146 T F T <--- 26 520-539 79 79 79 144 144 144 F F F 27 540-559 79 79 79 144 144 144 F F F 28 560-579 78 78 78 143 143 143 F F F 29 580-599 78 78 78 143 143 143 F F F Max gen: 82, Max flush: 154 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 150 150 150 F F F 1 020-039 82 82 82 150 150 150 F F F 2 040-059 82 82 82 154 154 154 F F F 3 060-079 81 81 81 149 149 149 F F F 4 080-099 80 80 80 148 148 148 F F F 5 100-119 83 83 83 155 155 155 F F F 6 120-139 83 83 83 155 155 155 F F F 7 140-159 82 82 82 150 150 150 F F F 8 160-179 82 82 82 154 154 154 F F F 9 180-199 81 81 81 149 149 149 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 83 83 83 155 155 155 F F F 12 240-259 83 83 83 155 155 155 F F F 13 260-279 81 81 81 149 149 149 F F F 14 280-299 81 81 81 149 149 149 F F F 15 300-319 82 82 82 153 153 153 F F F 16 320-339 80 80 80 147 147 147 F F F 17 340-359 80 80 80 146 146 146 F F F 18 360-379 72 72 72 134 134 134 F F F 19 380-399 82 82 82 151 151 151 F F F 20 400-419 82 82 82 151 151 151 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 83 83 83 155 155 155 F F F 23 460-479 82 82 82 150 150 150 F F F 24 480-499 82 82 82 150 150 150 F F F 25 500-519 83 83 83 155 155 155 F F F 26 520-539 79 79 79 144 144 144 F F F 27 540-559 79 79 79 144 144 144 F F F 28 560-579 78 78 78 143 143 143 F F F 29 580-599 78 78 78 143 143 143 F F F Max gen: 83, Max flush: 155 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.833932243Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.842118067Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.842422682Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.842447889Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.842456338Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.842467767Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475} {"msg":"Crucible ad41f856-ac1a-45c6-89f0-6c9998d8df94 has session id: 33aac10d-f743-4e4b-b995-03b2ebc299bb","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.84278316Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"Upstairs opts: Upstairs UUID: ad41f856-ac1a-45c6-89f0-6c9998d8df94, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.842828944Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"Crucible stats registered with UUID: ad41f856-ac1a-45c6-89f0-6c9998d8df94","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.842854332Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:14.84286449Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.842989266Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.843005024Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.843015303Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.843170055Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.843193912Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475} {"msg":"ad41f856-ac1a-45c6-89f0-6c9998d8df94 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.84320666Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.843217269Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"client":"0","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.843227098Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"client":"1","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.843236697Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"client":"2","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.843246146Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.843256085Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.843265873Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:40730)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.843355173Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"io task","client":"1","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:39975)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.843398698Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"io task","client":"2","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"ds_connection connected from Ok(127.0.0.1:34000)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.843444183Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"io task","client":"0","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844040233Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"client":"0","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844059491Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"client":"1","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.84407053Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"client":"2","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844364995Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.84440897Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"[0]R flush_numbers[0..12]: [150, 150, 154, 149, 148, 155, 155, 150, 154, 149, 142, 155]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844451965Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"downstairs","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"[0]R generation[0..12]: [82, 82, 82, 81, 80, 83, 83, 82, 82, 81, 78, 83]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844467123Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"downstairs","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844478532Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"downstairs","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"[1]R flush_numbers[0..12]: [150, 150, 154, 149, 148, 155, 155, 150, 154, 149, 142, 155]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844489211Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"downstairs","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"[1]R generation[0..12]: [82, 82, 82, 81, 80, 83, 83, 82, 82, 81, 78, 83]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844499679Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"downstairs","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844510518Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"downstairs","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"[2]R flush_numbers[0..12]: [150, 150, 154, 149, 148, 155, 155, 150, 154, 149, 142, 155]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844521117Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"downstairs","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"[2]R generation[0..12]: [82, 82, 82, 81, 80, 83, 83, 82, 82, 81, 78, 83]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844531296Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"downstairs","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844542055Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"downstairs","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844551773Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"downstairs","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844561432Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"downstairs","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844570931Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"downstairs","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.84458052Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"":"downstairs","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844590199Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844599148Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"client":"0","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844608797Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"client":"1","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844620005Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"client":"2","":"downstairs","session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844629624Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844638773Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"ad41f856-ac1a-45c6-89f0-6c9998d8df94 is now active with session: 33aac10d-f743-4e4b-b995-03b2ebc299bb","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844648702Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844657431Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"session_id":"33aac10d-f743-4e4b-b995-03b2ebc299bb"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.84466649Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475} {"msg":"Activated sub_volume ad41f856-ac1a-45c6-89f0-6c9998d8df94","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844674649Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844682848Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.844971504Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"task":"crutest"} 01/30 Write block 280 len 4096 data: 11 02/30 Flush 03/30 Write block 484 len 20480 data: 6 5 5 4 8 04/30 Read block 169 len 40960 05/30 Write block 265 len 36864 data: 5 6 8 8 9 10 8 9 9 06/30 Read block 194 len 4096 07/30 Write block 272 len 32768 data: 10 10 10 12 12 11 12 13 08/30 Write block 52 len 16384 data: 9 9 9 9 09/30 Write block 228 len 8192 data: 8 8 10/30 Read block 511 len 28672 11/30 Flush 12/30 Write block 80 len 20480 data: 9 6 8 10 9 13/30 Write block 144 len 20480 data: 6 7 7 8 8 14/30 Write block 239 len 32768 data: 12 10 11 12 11 12 10 10 15/30 Read block 355 len 40960 16/30 Read block 71 len 8192 17/30 Write block 538 len 28672 data: 7 7 6 7 7 7 10 18/30 Read block 551 len 24576 19/30 Write block 552 len 12288 data: 10 12 9 20/30 Read block 164 len 8192 21/30 Write block 321 len 40960 data: 10 7 5 6 5 3 4 7 7 8 22/30 Read block 198 len 36864 23/30 Flush 24/30 Write block 106 len 32768 data: 6 5 5 3 3 3 4 5 25/30 Read block 447 len 40960 26/30 Write block 116 len 36864 data: 7 8 9 10 9 9 10 11 11 27/30 Read block 312 len 32768 28/30 Read block 471 len 32768 29/30 Write block 27 len 8192 data: 11 13 30/30 Write block 522 len 36864 data: 4 4 4 4 3 3 3 4 6 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 5 Done Done Sent false 1003 Acked Read 10 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Read 1 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Write 4 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 8 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 29 32 Done 29 28 1 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:14.854050775Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:14.854232593Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1475,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 150 150 150 F F F 1 020-039 82 82 82 150 150 150 T T F <--- 2 040-059 84 84 82 157 157 154 F F F <--- 3 060-079 81 81 81 149 149 149 F F F 4 080-099 84 84 80 158 158 148 F F F <--- 5 100-119 83 83 83 155 155 155 T T F <--- 6 120-139 83 83 83 155 155 155 T T F <--- 7 140-159 84 84 82 158 158 150 F F F <--- 8 160-179 82 82 82 154 154 154 F F F 9 180-199 81 81 81 149 149 149 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 84 84 83 158 158 155 F F F <--- 12 240-259 84 84 83 158 158 155 F F F <--- 13 260-279 84 84 81 157 157 149 F F F <--- 14 280-299 84 84 81 156 156 149 F F T <--- 15 300-319 82 82 82 153 153 153 F F F 16 320-339 84 84 80 158 158 147 F F F <--- 17 340-359 80 80 80 146 146 146 F F F 18 360-379 72 72 72 134 134 134 F F F 19 380-399 82 82 82 151 151 151 F F F 20 400-419 82 82 82 151 151 151 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 83 83 83 155 155 155 F F F 23 460-479 82 82 82 150 150 150 F F F 24 480-499 84 84 82 157 157 150 F F F <--- 25 500-519 83 83 83 155 155 155 F F F 26 520-539 84 84 79 158 158 144 T T F <--- 27 540-559 84 84 79 158 158 144 F F F <--- 28 560-579 78 78 78 143 143 143 F F F 29 580-599 78 78 78 143 143 143 F F F Max gen: 84, Max flush: 158 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 150 150 150 F F F 1 020-039 85 85 85 159 159 159 F F F 2 040-059 84 84 84 157 157 157 F F F 3 060-079 81 81 81 149 149 149 F F F 4 080-099 84 84 84 158 158 158 F F F 5 100-119 85 85 85 159 159 159 F F F 6 120-139 85 85 85 159 159 159 F F F 7 140-159 84 84 84 158 158 158 F F F 8 160-179 82 82 82 154 154 154 F F F 9 180-199 81 81 81 149 149 149 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 84 84 84 158 158 158 F F F 12 240-259 84 84 84 158 158 158 F F F 13 260-279 84 84 84 157 157 157 F F F 14 280-299 84 84 84 156 156 156 F F F 15 300-319 82 82 82 153 153 153 F F F 16 320-339 84 84 84 158 158 158 F F F 17 340-359 80 80 80 146 146 146 F F F 18 360-379 72 72 72 134 134 134 F F F 19 380-399 82 82 82 151 151 151 F F F 20 400-419 82 82 82 151 151 151 F F F 21 420-439 74 74 74 138 138 138 F F F 22 440-459 83 83 83 155 155 155 F F F 23 460-479 82 82 82 150 150 150 F F F 24 480-499 84 84 84 157 157 157 F F F 25 500-519 83 83 83 155 155 155 F F F 26 520-539 85 85 85 159 159 159 F F F 27 540-559 84 84 84 158 158 158 F F F 28 560-579 78 78 78 143 143 143 F F F 29 580-599 78 78 78 143 143 143 F F F Max gen: 85, Max flush: 159 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.107252783Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:19.11438304Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.114438194Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.121388993Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.121640723Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.121675549Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.121687638Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.121699486Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483} {"msg":"Crucible c9794f8b-7c16-40be-a3b6-d18dee199edb has session id: 3ff9d73b-35ef-4a09-abd9-8aaf410c0138","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122017269Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"Upstairs opts: Upstairs UUID: c9794f8b-7c16-40be-a3b6-d18dee199edb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122053525Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"Crucible stats registered with UUID: c9794f8b-7c16-40be-a3b6-d18dee199edb","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122064044Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:19.122073203Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122216906Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122246542Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122257521Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122397855Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122408864Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483} {"msg":"c9794f8b-7c16-40be-a3b6-d18dee199edb active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.12244133Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122450989Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"client":"0","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122460937Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"client":"1","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122470626Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"client":"2","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122482535Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122492624Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122502573Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"ds_connection connected from Ok(127.0.0.1:41175)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122551387Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"io task","client":"1","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"ds_connection connected from Ok(127.0.0.1:33616)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122571725Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"io task","client":"0","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"ds_connection connected from Ok(127.0.0.1:45203)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.122646326Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"io task","client":"2","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123150867Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"client":"1","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123172714Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"client":"2","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123222319Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"client":"0","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123441543Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123474289Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"[0]R flush_numbers[0..12]: [150, 159, 157, 149, 158, 159, 159, 158, 154, 149, 142, 158]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123524373Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"downstairs","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"[0]R generation[0..12]: [82, 85, 84, 81, 84, 85, 85, 84, 82, 81, 78, 84]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123535522Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"downstairs","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123546681Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"downstairs","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"[1]R flush_numbers[0..12]: [150, 159, 157, 149, 158, 159, 159, 158, 154, 149, 142, 158]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123557219Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"downstairs","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"[1]R generation[0..12]: [82, 85, 84, 81, 84, 85, 85, 84, 82, 81, 78, 84]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123570038Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"downstairs","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123580907Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"downstairs","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"[2]R flush_numbers[0..12]: [150, 159, 157, 149, 158, 159, 159, 158, 154, 149, 142, 158]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123591385Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"downstairs","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"[2]R generation[0..12]: [82, 85, 84, 81, 84, 85, 85, 84, 82, 81, 78, 84]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123601724Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"downstairs","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123612653Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"downstairs","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123622082Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"downstairs","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123631781Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"downstairs","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.1236412Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"downstairs","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123651029Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"":"downstairs","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123660877Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123669536Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"client":"0","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123679165Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"client":"1","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123688724Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"client":"2","":"downstairs","session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123698603Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123707692Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"c9794f8b-7c16-40be-a3b6-d18dee199edb is now active with session: 3ff9d73b-35ef-4a09-abd9-8aaf410c0138","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123718831Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.12372751Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"session_id":"3ff9d73b-35ef-4a09-abd9-8aaf410c0138"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123736729Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483} {"msg":"Activated sub_volume c9794f8b-7c16-40be-a3b6-d18dee199edb","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123744858Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.123752977Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.124061431Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"task":"crutest"} 01/30 Flush 02/30 Write block 434 len 36864 data: 9 8 8 5 5 7 7 10 10 03/30 Read block 61 len 24576 04/30 Read block 520 len 4096 05/30 Write block 295 len 32768 data: 10 9 9 10 9 10 8 9 06/30 Read block 107 len 20480 07/30 Write block 378 len 36864 data: 4 4 6 6 8 9 7 5 6 08/30 Write block 593 len 28672 data: 5 4 5 5 3 3 2 09/30 Read block 214 len 8192 10/30 Flush 11/30 Write block 538 len 28672 data: 8 8 7 8 8 8 11 12/30 Read block 532 len 20480 13/30 Read block 413 len 40960 14/30 Flush 15/30 Write block 593 len 12288 data: 6 5 6 16/30 Read block 348 len 8192 17/30 Write block 100 len 20480 data: 6 5 5 4 5 18/30 Read block 398 len 16384 19/30 Read block 521 len 16384 20/30 Read block 118 len 12288 21/30 Write block 222 len 32768 data: 6 7 7 7 8 9 9 9 22/30 Write block 152 len 40960 data: 11 10 8 8 7 8 8 12 11 8 23/30 Flush 24/30 Write block 176 len 20480 data: 6 5 6 6 6 25/30 Write block 48 len 36864 data: 6 7 6 11 10 10 10 10 11 26/30 Write block 541 len 32768 data: 9 9 9 12 10 11 11 10 27/30 Write block 27 len 32768 data: 12 14 14 14 16 16 14 11 28/30 Read block 492 len 40960 29/30 Read block 423 len 8192 30/30 Write block 568 len 40960 data: 7 3 5 8 9 8 9 9 9 10 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Sent Done Done false 1001 Acked Write 9 Sent Done Done false 1002 Acked Read 6 Sent Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Write 7 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 3 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:19.133735301Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:19.133925929Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1483,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 150 150 150 F F F 1 020-039 85 85 85 159 159 159 F T T <--- 2 040-059 84 84 84 157 157 157 F T T <--- 3 060-079 81 81 81 149 149 149 F F F 4 080-099 84 84 84 158 158 158 F F F 5 100-119 85 86 86 159 163 163 F F F <--- 6 120-139 85 85 85 159 159 159 F F F 7 140-159 84 86 86 158 163 163 F F F <--- 8 160-179 82 86 86 154 163 163 F T T <--- 9 180-199 81 81 81 149 149 149 F T T <--- 10 200-219 78 78 78 142 142 142 F F F 11 220-239 84 86 86 158 163 163 F F F <--- 12 240-259 84 84 84 158 158 158 F F F 13 260-279 84 84 84 157 157 157 F F F 14 280-299 84 86 86 156 161 161 F F F <--- 15 300-319 82 86 86 153 161 161 F F F <--- 16 320-339 84 84 84 158 158 158 F F F 17 340-359 80 80 80 146 146 146 F F F 18 360-379 72 86 86 134 161 161 F F F <--- 19 380-399 82 86 86 151 161 161 F F F <--- 20 400-419 82 82 82 151 151 151 F F F 21 420-439 74 86 86 138 161 161 F F F <--- 22 440-459 83 86 86 155 161 161 F F F <--- 23 460-479 82 82 82 150 150 150 F F F 24 480-499 84 84 84 157 157 157 F F F 25 500-519 83 83 83 155 155 155 F F F 26 520-539 85 86 86 159 162 162 F F F <--- 27 540-559 84 86 86 158 162 162 F T T <--- 28 560-579 78 78 78 143 143 143 F T T <--- 29 580-599 78 86 86 143 163 163 F F F <--- Max gen: 86, Max flush: 163 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 150 150 150 F F F 1 020-039 87 87 87 164 164 164 F F F 2 040-059 87 87 87 164 164 164 F F F 3 060-079 81 81 81 149 149 149 F F F 4 080-099 84 84 84 158 158 158 F F F 5 100-119 86 86 86 163 163 163 F F F 6 120-139 85 85 85 159 159 159 F F F 7 140-159 86 86 86 163 163 163 F F F 8 160-179 87 87 87 164 164 164 F F F 9 180-199 87 87 87 164 164 164 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 86 86 86 163 163 163 F F F 12 240-259 84 84 84 158 158 158 F F F 13 260-279 84 84 84 157 157 157 F F F 14 280-299 86 86 86 161 161 161 F F F 15 300-319 86 86 86 161 161 161 F F F 16 320-339 84 84 84 158 158 158 F F F 17 340-359 80 80 80 146 146 146 F F F 18 360-379 86 86 86 161 161 161 F F F 19 380-399 86 86 86 161 161 161 F F F 20 400-419 82 82 82 151 151 151 F F F 21 420-439 86 86 86 161 161 161 F F F 22 440-459 86 86 86 161 161 161 F F F 23 460-479 82 82 82 150 150 150 F F F 24 480-499 84 84 84 157 157 157 F F F 25 500-519 83 83 83 155 155 155 F F F 26 520-539 86 86 86 162 162 162 F F F 27 540-559 87 87 87 164 164 164 F F F 28 560-579 87 87 87 164 164 164 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 87, Max flush: 164 Begin loop 44 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.4002957Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.408798498Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409091713Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.40911796Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409126549Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409137868Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491} {"msg":"Crucible 0722e38e-a9db-4d97-8240-737d1669bdda has session id: e63bb52b-63bc-454b-8c0f-291cc3632481","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409425694Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"Upstairs opts: Upstairs UUID: 0722e38e-a9db-4d97-8240-737d1669bdda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409454381Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"Crucible stats registered with UUID: 0722e38e-a9db-4d97-8240-737d1669bdda","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.40946758Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:23.409477138Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409611453Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409641189Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409652068Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409781903Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409800051Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491} {"msg":"0722e38e-a9db-4d97-8240-737d1669bdda active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409824858Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409834377Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"client":"0","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409844266Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"client":"1","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409853785Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"client":"2","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409863373Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409873292Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409883111Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"ds_connection connected from Ok(127.0.0.1:53590)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409927996Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"io task","client":"2","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"ds_connection connected from Ok(127.0.0.1:56692)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.409938935Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"io task","client":"1","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"ds_connection connected from Ok(127.0.0.1:44594)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.41006538Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"io task","client":"0","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.410620395Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"client":"1","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.410639073Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"client":"0","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.410677758Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"client":"2","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.410934588Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.410944957Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"[0]R flush_numbers[0..12]: [150, 164, 164, 149, 158, 163, 159, 163, 164, 164, 142, 163]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.4110057Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"downstairs","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"[0]R generation[0..12]: [82, 87, 87, 81, 84, 86, 85, 86, 87, 87, 78, 86]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411017359Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"downstairs","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411028607Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"downstairs","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"[1]R flush_numbers[0..12]: [150, 164, 164, 149, 158, 163, 159, 163, 164, 164, 142, 163]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411039166Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"downstairs","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"[1]R generation[0..12]: [82, 87, 87, 81, 84, 86, 85, 86, 87, 87, 78, 86]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411049455Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"downstairs","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411060304Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"downstairs","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"[2]R flush_numbers[0..12]: [150, 164, 164, 149, 158, 163, 159, 163, 164, 164, 142, 163]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411070792Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"downstairs","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"[2]R generation[0..12]: [82, 87, 87, 81, 84, 86, 85, 86, 87, 87, 78, 86]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411081011Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"downstairs","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.41109485Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"downstairs","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411104619Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"downstairs","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411114207Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"downstairs","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411123576Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"downstairs","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411132985Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"":"downstairs","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411142664Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411151493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"client":"0","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411161152Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"client":"1","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411170711Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"client":"2","":"downstairs","session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.4111805Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411189559Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"0722e38e-a9db-4d97-8240-737d1669bdda is now active with session: e63bb52b-63bc-454b-8c0f-291cc3632481","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411199517Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411208376Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"session_id":"e63bb52b-63bc-454b-8c0f-291cc3632481"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411217485Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491} {"msg":"Activated sub_volume 0722e38e-a9db-4d97-8240-737d1669bdda","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411225744Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.411234013Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.4115214Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"task":"crutest"} 01/30 Write block 151 len 36864 data: 10 12 11 9 9 8 9 9 13 02/30 Flush 03/30 Write block 141 len 24576 data: 5 6 5 7 8 8 04/30 Write block 471 len 28672 data: 3 5 7 8 7 9 9 05/30 Write block 15 len 32768 data: 5 5 5 4 6 7 5 7 06/30 Read block 157 len 20480 07/30 Read block 483 len 32768 08/30 Write block 503 len 20480 data: 4 4 5 7 8 09/30 Write block 371 len 28672 data: 4 3 2 3 6 6 4 10/30 Write block 92 len 20480 data: 10 10 10 9 8 11/30 Read block 414 len 20480 12/30 Read block 465 len 36864 13/30 Read block 121 len 4096 14/30 Read block 571 len 20480 15/30 Write block 457 len 16384 data: 7 7 7 6 16/30 Write block 189 len 12288 data: 9 10 11 17/30 Write block 304 len 36864 data: 8 12 11 10 8 7 5 2 2 18/30 Write block 417 len 40960 data: 4 5 5 7 6 5 5 4 4 7 19/30 Write block 511 len 28672 data: 6 6 8 9 10 10 9 20/30 Read block 101 len 4096 21/30 Write block 191 len 24576 data: 12 9 9 9 9 8 22/30 Write block 419 len 12288 data: 6 8 7 23/30 Write block 546 len 12288 data: 12 12 11 24/30 Read block 547 len 12288 25/30 Flush 26/30 Write block 4 len 4096 data: 9 27/30 Read block 169 len 12288 28/30 Read block 519 len 16384 29/30 Read block 501 len 12288 30/30 Read block 270 len 16384 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 6 Done Done Sent false 1003 Acked Write 7 Done Done Sent false 1004 Acked Write 8 Done Done Sent false 1005 Acked Read 5 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Write 7 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:23.419752529Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:23.41991638Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1491,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 88 88 82 166 166 150 T T F <--- 1 020-039 88 88 87 166 166 164 F F F <--- 2 040-059 87 87 87 164 164 164 F F F 3 060-079 81 81 81 149 149 149 F F F 4 080-099 88 88 84 166 166 158 F F F <--- 5 100-119 86 86 86 163 163 163 F F F 6 120-139 85 85 85 159 159 159 F F F 7 140-159 88 88 86 166 166 163 F F T <--- 8 160-179 87 87 87 164 164 164 F F F 9 180-199 88 88 87 166 166 164 F F F <--- 10 200-219 78 78 78 142 142 142 F F F 11 220-239 86 86 86 163 163 163 F F F 12 240-259 84 84 84 158 158 158 F F F 13 260-279 84 84 84 157 157 157 F F F 14 280-299 86 86 86 161 161 161 F F F 15 300-319 88 88 86 166 166 161 F F F <--- 16 320-339 84 84 84 158 158 158 F F F 17 340-359 80 80 80 146 146 146 F F F 18 360-379 88 88 86 166 166 161 F F F <--- 19 380-399 86 86 86 161 161 161 F F F 20 400-419 88 88 82 166 166 151 F F F <--- 21 420-439 88 88 86 166 166 161 F F F <--- 22 440-459 88 88 86 166 166 161 F F F <--- 23 460-479 88 88 82 166 166 150 F F F <--- 24 480-499 84 84 84 157 157 157 F F F 25 500-519 88 88 83 166 166 155 F F F <--- 26 520-539 86 86 86 162 162 162 F F F 27 540-559 88 88 87 166 166 164 F F F <--- 28 560-579 87 87 87 164 164 164 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 88, Max flush: 166 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 167 167 167 F F F 1 020-039 88 88 88 166 166 166 F F F 2 040-059 87 87 87 164 164 164 F F F 3 060-079 81 81 81 149 149 149 F F F 4 080-099 88 88 88 166 166 166 F F F 5 100-119 86 86 86 163 163 163 F F F 6 120-139 85 85 85 159 159 159 F F F 7 140-159 88 88 88 166 166 166 F F F 8 160-179 87 87 87 164 164 164 F F F 9 180-199 88 88 88 166 166 166 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 86 86 86 163 163 163 F F F 12 240-259 84 84 84 158 158 158 F F F 13 260-279 84 84 84 157 157 157 F F F 14 280-299 86 86 86 161 161 161 F F F 15 300-319 88 88 88 166 166 166 F F F 16 320-339 84 84 84 158 158 158 F F F 17 340-359 80 80 80 146 146 146 F F F 18 360-379 88 88 88 166 166 166 F F F 19 380-399 86 86 86 161 161 161 F F F 20 400-419 88 88 88 166 166 166 F F F 21 420-439 88 88 88 166 166 166 F F F 22 440-459 88 88 88 166 166 166 F F F 23 460-479 88 88 88 166 166 166 F F F 24 480-499 84 84 84 157 157 157 F F F 25 500-519 88 88 88 166 166 166 F F F 26 520-539 86 86 86 162 162 162 F F F 27 540-559 88 88 88 166 166 166 F F F 28 560-579 87 87 87 164 164 164 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 89, Max flush: 167 Begin loop 45 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.668169046Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:27.675352327Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.675401782Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.682212017Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.682463557Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.682489464Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.682497893Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.682509362Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499} {"msg":"Crucible 0ee3d0f5-7b75-45c8-b0fb-961facf27192 has session id: 1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.682823255Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"Upstairs opts: Upstairs UUID: 0ee3d0f5-7b75-45c8-b0fb-961facf27192, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.682844553Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"Crucible stats registered with UUID: 0ee3d0f5-7b75-45c8-b0fb-961facf27192","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.682854412Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:27.682863501Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683011023Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.68304178Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683052958Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683185883Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683201611Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499} {"msg":"0ee3d0f5-7b75-45c8-b0fb-961facf27192 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683214839Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683223648Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"client":"0","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683233297Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"client":"1","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683242926Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"client":"2","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683252455Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683262324Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683272233Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"ds_connection connected from Ok(127.0.0.1:33498)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683313498Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"io task","client":"0","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"ds_connection connected from Ok(127.0.0.1:41608)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683367932Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"io task","client":"2","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"ds_connection connected from Ok(127.0.0.1:58245)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.68337897Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"io task","client":"1","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.683968662Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"client":"2","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684052122Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"client":"1","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.68415329Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"client":"0","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684259728Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.68432435Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"[0]R flush_numbers[0..12]: [167, 166, 164, 149, 166, 163, 159, 166, 164, 166, 142, 163]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.68441224Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"downstairs","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"[0]R generation[0..12]: [89, 88, 87, 81, 88, 86, 85, 88, 87, 88, 78, 86]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684423388Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"downstairs","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684434437Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"downstairs","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"[1]R flush_numbers[0..12]: [167, 166, 164, 149, 166, 163, 159, 166, 164, 166, 142, 163]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684444976Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"downstairs","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"[1]R generation[0..12]: [89, 88, 87, 81, 88, 86, 85, 88, 87, 88, 78, 86]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684455335Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"downstairs","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684466263Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"downstairs","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"[2]R flush_numbers[0..12]: [167, 166, 164, 149, 166, 163, 159, 166, 164, 166, 142, 163]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684476832Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"downstairs","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"[2]R generation[0..12]: [89, 88, 87, 81, 88, 86, 85, 88, 87, 88, 78, 86]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684487071Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"downstairs","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.68449787Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"downstairs","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684507279Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"downstairs","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684516768Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"downstairs","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684528136Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"downstairs","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684537425Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"":"downstairs","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684547414Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684556253Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"client":"0","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684566092Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"client":"1","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684575971Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"client":"2","":"downstairs","session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684585919Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684595048Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"0ee3d0f5-7b75-45c8-b0fb-961facf27192 is now active with session: 1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684605067Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684613696Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"session_id":"1b3dd79e-2ae7-4ca5-ae04-f32ed8bd1edd"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684622745Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499} {"msg":"Activated sub_volume 0ee3d0f5-7b75-45c8-b0fb-961facf27192","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684630714Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684638833Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.684940138Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"task":"crutest"} 01/30 Write block 543 len 36864 data: 10 13 11 13 13 12 12 12 10 02/30 Read block 82 len 20480 03/30 Read block 16 len 8192 04/30 Flush 05/30 Write block 30 len 4096 data: 15 06/30 Read block 177 len 32768 07/30 Write block 241 len 8192 data: 12 13 08/30 Read block 33 len 12288 09/30 Read block 65 len 20480 10/30 Write block 518 len 32768 data: 10 9 7 5 5 5 5 5 11/30 Read block 245 len 28672 12/30 Write block 56 len 12288 data: 12 8 7 13/30 Read block 290 len 4096 14/30 Read block 50 len 36864 15/30 Write block 300 len 32768 data: 11 9 10 11 9 13 12 11 16/30 Read block 590 len 32768 17/30 Read block 495 len 16384 18/30 Write block 147 len 12288 data: 9 9 8 19/30 Write block 346 len 40960 data: 5 4 5 7 6 8 10 10 11 7 20/30 Write block 530 len 32768 data: 7 6 8 9 8 8 9 7 21/30 Read block 468 len 16384 22/30 Flush 23/30 Write block 38 len 12288 data: 4 4 6 24/30 Flush 25/30 Write block 223 len 12288 data: 8 8 8 26/30 Read block 322 len 36864 27/30 Read block 280 len 36864 28/30 Read block 576 len 20480 29/30 Write block 257 len 4096 data: 5 30/30 Read block 120 len 20480 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Sent Done Done false 1001 Acked Read 5 Sent Done Done false 1002 Acked Read 2 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:27.693572051Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:27.693752909Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1499,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 167 167 167 F F F 1 020-039 88 90 90 166 170 170 F F F <--- 2 040-059 87 90 90 164 170 170 F F F <--- 3 060-079 81 81 81 149 149 149 F F F 4 080-099 88 88 88 166 166 166 F F F 5 100-119 86 86 86 163 163 163 F F F 6 120-139 85 85 85 159 159 159 F F F 7 140-159 88 90 90 166 169 169 F F F <--- 8 160-179 87 87 87 164 164 164 F F F 9 180-199 88 88 88 166 166 166 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 86 86 86 163 163 163 F T T <--- 12 240-259 84 90 90 158 169 169 F T T <--- 13 260-279 84 84 84 157 157 157 F F F 14 280-299 86 86 86 161 161 161 F F F 15 300-319 88 90 90 166 169 169 F F F <--- 16 320-339 84 84 84 158 158 158 F F F 17 340-359 80 90 90 146 169 169 F F F <--- 18 360-379 88 88 88 166 166 166 F F F 19 380-399 86 86 86 161 161 161 F F F 20 400-419 88 88 88 166 166 166 F F F 21 420-439 88 88 88 166 166 166 F F F 22 440-459 88 88 88 166 166 166 F F F 23 460-479 88 88 88 166 166 166 F F F 24 480-499 84 84 84 157 157 157 F F F 25 500-519 88 90 90 166 169 169 F F F <--- 26 520-539 86 90 90 162 169 169 F F F <--- 27 540-559 88 90 90 166 168 168 F F F <--- 28 560-579 87 87 87 164 164 164 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 90, Max flush: 170 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 167 167 167 F F F 1 020-039 90 90 90 170 170 170 F F F 2 040-059 90 90 90 170 170 170 F F F 3 060-079 81 81 81 149 149 149 F F F 4 080-099 88 88 88 166 166 166 F F F 5 100-119 86 86 86 163 163 163 F F F 6 120-139 85 85 85 159 159 159 F F F 7 140-159 90 90 90 169 169 169 F F F 8 160-179 87 87 87 164 164 164 F F F 9 180-199 88 88 88 166 166 166 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 91 91 91 171 171 171 F F F 12 240-259 91 91 91 171 171 171 F F F 13 260-279 84 84 84 157 157 157 F F F 14 280-299 86 86 86 161 161 161 F F F 15 300-319 90 90 90 169 169 169 F F F 16 320-339 84 84 84 158 158 158 F F F 17 340-359 90 90 90 169 169 169 F F F 18 360-379 88 88 88 166 166 166 F F F 19 380-399 86 86 86 161 161 161 F F F 20 400-419 88 88 88 166 166 166 F F F 21 420-439 88 88 88 166 166 166 F F F 22 440-459 88 88 88 166 166 166 F F F 23 460-479 88 88 88 166 166 166 F F F 24 480-499 84 84 84 157 157 157 F F F 25 500-519 90 90 90 169 169 169 F F F 26 520-539 90 90 90 169 169 169 F F F 27 540-559 90 90 90 168 168 168 F F F 28 560-579 87 87 87 164 164 164 F F F 29 580-599 86 86 86 163 163 163 F F F Max gen: 91, Max flush: 171 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.942098354Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:31.9491544Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.949209444Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.955890494Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956143204Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.95617857Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956187929Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956199738Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507} {"msg":"Crucible e72ccfe8-9937-4bf7-9882-c91644c1b97a has session id: 2ad86475-c99b-4ad0-8da0-12d22ed85be9","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956492604Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"Upstairs opts: Upstairs UUID: e72ccfe8-9937-4bf7-9882-c91644c1b97a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.95652395Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"Crucible stats registered with UUID: e72ccfe8-9937-4bf7-9882-c91644c1b97a","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956535099Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:31.956544408Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956688321Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956705399Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956715728Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956830704Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956849562Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507} {"msg":"e72ccfe8-9937-4bf7-9882-c91644c1b97a active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.95686604Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956876799Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"client":"0","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956886668Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"client":"1","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956896277Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"client":"2","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956912725Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956922913Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.956932842Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"ds_connection connected from Ok(127.0.0.1:34373)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.957052288Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"io task","client":"1","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"ds_connection connected from Ok(127.0.0.1:41342)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.957077415Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"io task","client":"0","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"ds_connection connected from Ok(127.0.0.1:64135)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.957088644Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"io task","client":"2","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.957687274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"client":"2","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.957707082Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"client":"1","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.95772579Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"client":"0","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958000158Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958010037Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"[0]R flush_numbers[0..12]: [167, 170, 170, 149, 166, 163, 159, 169, 164, 166, 142, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958034874Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"downstairs","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"[0]R generation[0..12]: [89, 90, 90, 81, 88, 86, 85, 90, 87, 88, 78, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958048222Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"downstairs","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958059321Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"downstairs","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"[1]R flush_numbers[0..12]: [167, 170, 170, 149, 166, 163, 159, 169, 164, 166, 142, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.95806986Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"downstairs","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"[1]R generation[0..12]: [89, 90, 90, 81, 88, 86, 85, 90, 87, 88, 78, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958080068Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"downstairs","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958090777Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"downstairs","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"[2]R flush_numbers[0..12]: [167, 170, 170, 149, 166, 163, 159, 169, 164, 166, 142, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958101336Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"downstairs","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"[2]R generation[0..12]: [89, 90, 90, 81, 88, 86, 85, 90, 87, 88, 78, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958111635Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"downstairs","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958122553Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"downstairs","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958131892Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"downstairs","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958141541Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"downstairs","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.95815092Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"downstairs","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958160219Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"":"downstairs","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958170108Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958178877Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"client":"0","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958189736Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"client":"1","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958199494Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"client":"2","":"downstairs","session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958209503Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958218672Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"e72ccfe8-9937-4bf7-9882-c91644c1b97a is now active with session: 2ad86475-c99b-4ad0-8da0-12d22ed85be9","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958228531Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.95823742Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"session_id":"2ad86475-c99b-4ad0-8da0-12d22ed85be9"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958246349Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507} {"msg":"Activated sub_volume e72ccfe8-9937-4bf7-9882-c91644c1b97a","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958254318Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958262327Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.958552273Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"task":"crutest"} 01/30 Write block 21 len 4096 data: 6 02/30 Write block 118 len 32768 data: 10 11 10 10 11 12 12 11 03/30 Write block 37 len 20480 data: 5 5 5 7 6 04/30 Flush 05/30 Write block 429 len 32768 data: 7 7 8 9 10 10 9 9 06/30 Read block 115 len 20480 07/30 Write block 335 len 36864 data: 10 9 9 7 8 8 5 5 6 08/30 Read block 305 len 28672 09/30 Write block 576 len 28672 data: 10 11 7 9 8 9 8 10/30 Read block 120 len 40960 11/30 Read block 443 len 20480 12/30 Write block 497 len 20480 data: 7 5 6 7 6 13/30 Read block 432 len 16384 14/30 Write block 337 len 12288 data: 10 8 9 15/30 Write block 540 len 36864 data: 8 10 10 11 14 12 14 14 13 16/30 Read block 226 len 28672 17/30 Write block 376 len 20480 data: 7 5 5 5 7 18/30 Read block 410 len 8192 19/30 Write block 458 len 32768 data: 8 8 7 5 6 6 6 5 20/30 Read block 25 len 24576 21/30 Read block 167 len 40960 22/30 Read block 38 len 24576 23/30 Write block 310 len 32768 data: 6 3 3 3 3 5 7 9 24/30 Flush 25/30 Write block 412 len 16384 data: 6 5 5 5 26/30 Read block 425 len 8192 27/30 Read block 145 len 28672 28/30 Write block 161 len 24576 data: 9 6 7 8 8 8 29/30 Write block 114 len 32768 data: 6 6 8 9 11 12 11 11 30/30 Read block 232 len 8192 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Sent Done Done false 1001 Acked Write 8 Sent Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Write 8 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:31.968230954Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:31.968396544Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1507,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 167 167 167 F F F 1 020-039 90 92 92 170 172 172 F F F <--- 2 040-059 90 92 92 170 172 172 F F F <--- 3 060-079 81 81 81 149 149 149 F F F 4 080-099 88 88 88 166 166 166 F F F 5 100-119 86 92 92 163 172 172 F T T <--- 6 120-139 85 92 92 159 172 172 F T T <--- 7 140-159 90 90 90 169 169 169 F F F 8 160-179 87 87 87 164 164 164 F T T <--- 9 180-199 88 88 88 166 166 166 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 91 91 91 171 171 171 F F F 12 240-259 91 91 91 171 171 171 F F F 13 260-279 84 84 84 157 157 157 F F F 14 280-299 86 86 86 161 161 161 F F F 15 300-319 90 92 92 169 173 173 F F F <--- 16 320-339 84 92 92 158 173 173 F F F <--- 17 340-359 90 92 92 169 173 173 F F F <--- 18 360-379 88 92 92 166 173 173 F F F <--- 19 380-399 86 92 92 161 173 173 F F F <--- 20 400-419 88 88 88 166 166 166 F T T <--- 21 420-439 88 92 92 166 173 173 F F F <--- 22 440-459 88 92 92 166 173 173 F F F <--- 23 460-479 88 92 92 166 173 173 F F F <--- 24 480-499 84 92 92 157 173 173 F F F <--- 25 500-519 90 92 92 169 173 173 F F F <--- 26 520-539 90 90 90 169 169 169 F F F 27 540-559 90 92 92 168 173 173 F F F <--- 28 560-579 87 92 92 164 173 173 F F F <--- 29 580-599 86 92 92 163 173 173 F F F <--- Max gen: 92, Max flush: 173 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 167 167 167 F F F 1 020-039 92 92 92 172 172 172 F F F 2 040-059 92 92 92 172 172 172 F F F 3 060-079 81 81 81 149 149 149 F F F 4 080-099 88 88 88 166 166 166 F F F 5 100-119 93 93 93 174 174 174 F F F 6 120-139 93 93 93 174 174 174 F F F 7 140-159 90 90 90 169 169 169 F F F 8 160-179 93 93 93 174 174 174 F F F 9 180-199 88 88 88 166 166 166 F F F 10 200-219 78 78 78 142 142 142 F F F 11 220-239 91 91 91 171 171 171 F F F 12 240-259 91 91 91 171 171 171 F F F 13 260-279 84 84 84 157 157 157 F F F 14 280-299 86 86 86 161 161 161 F F F 15 300-319 92 92 92 173 173 173 F F F 16 320-339 92 92 92 173 173 173 F F F 17 340-359 92 92 92 173 173 173 F F F 18 360-379 92 92 92 173 173 173 F F F 19 380-399 92 92 92 173 173 173 F F F 20 400-419 93 93 93 174 174 174 F F F 21 420-439 92 92 92 173 173 173 F F F 22 440-459 92 92 92 173 173 173 F F F 23 460-479 92 92 92 173 173 173 F F F 24 480-499 92 92 92 173 173 173 F F F 25 500-519 92 92 92 173 173 173 F F F 26 520-539 90 90 90 169 169 169 F F F 27 540-559 92 92 92 173 173 173 F F F 28 560-579 92 92 92 173 173 173 F F F 29 580-599 92 92 92 173 173 173 F F F Max gen: 93, Max flush: 174 Begin loop 47 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.233919004Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:36.241096066Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.24115088Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.247906941Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248170801Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248197727Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248208986Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248220495Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515} {"msg":"Crucible 5848e540-80f7-4468-a058-99898e97cfb2 has session id: 8e95b696-0fcb-45de-8640-4509ad4efbba","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248541777Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"Upstairs opts: Upstairs UUID: 5848e540-80f7-4468-a058-99898e97cfb2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248571614Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"Crucible stats registered with UUID: 5848e540-80f7-4468-a058-99898e97cfb2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248582792Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:36.248591941Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248718187Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248748143Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248759322Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248901935Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248924963Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515} {"msg":"5848e540-80f7-4468-a058-99898e97cfb2 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248934061Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.24894317Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"client":"0","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248953509Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"client":"1","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248963518Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"client":"2","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248975147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248985216Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.248995234Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"ds_connection connected from Ok(127.0.0.1:38911)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.2491167Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"io task","client":"1","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"ds_connection connected from Ok(127.0.0.1:40702)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.249187812Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"io task","client":"2","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"ds_connection connected from Ok(127.0.0.1:49512)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.2492014Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"io task","client":"0","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.249695713Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"client":"1","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.24971791Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"client":"2","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.249768354Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"client":"0","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250020425Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250030453Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"[0]R flush_numbers[0..12]: [167, 172, 172, 149, 166, 174, 174, 169, 174, 166, 142, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250070339Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"downstairs","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"[0]R generation[0..12]: [89, 92, 92, 81, 88, 93, 93, 90, 93, 88, 78, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250081708Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"downstairs","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250092686Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"downstairs","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"[1]R flush_numbers[0..12]: [167, 172, 172, 149, 166, 174, 174, 169, 174, 166, 142, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250103355Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"downstairs","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"[1]R generation[0..12]: [89, 92, 92, 81, 88, 93, 93, 90, 93, 88, 78, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250115674Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"downstairs","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250126812Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"downstairs","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"[2]R flush_numbers[0..12]: [167, 172, 172, 149, 166, 174, 174, 169, 174, 166, 142, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250137341Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"downstairs","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"[2]R generation[0..12]: [89, 92, 92, 81, 88, 93, 93, 90, 93, 88, 78, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.25014749Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"downstairs","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250158359Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"downstairs","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250167787Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"downstairs","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250177446Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"downstairs","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250186725Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"downstairs","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250196294Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"":"downstairs","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250206113Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250214952Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"client":"0","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250224721Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"client":"1","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.25023435Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"client":"2","":"downstairs","session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250244309Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250253257Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"5848e540-80f7-4468-a058-99898e97cfb2 is now active with session: 8e95b696-0fcb-45de-8640-4509ad4efbba","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250264526Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250273335Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"session_id":"8e95b696-0fcb-45de-8640-4509ad4efbba"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250282374Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515} {"msg":"Activated sub_volume 5848e540-80f7-4468-a058-99898e97cfb2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250290853Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250298972Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.250564631Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"task":"crutest"} 01/30 Write block 315 len 36864 data: 6 8 10 10 12 10 11 8 6 02/30 Read block 305 len 12288 03/30 Write block 457 len 4096 data: 8 04/30 Write block 256 len 24576 data: 5 6 3 3 4 4 05/30 Write block 547 len 36864 data: 15 14 13 13 11 11 13 10 9 06/30 Read block 306 len 40960 07/30 Read block 252 len 12288 08/30 Read block 305 len 36864 09/30 Write block 297 len 4096 data: 10 10/30 Read block 542 len 36864 11/30 Read block 422 len 32768 12/30 Read block 131 len 20480 13/30 Read block 476 len 8192 14/30 Read block 73 len 12288 15/30 Read block 323 len 24576 16/30 Write block 321 len 24576 data: 12 9 7 7 6 4 17/30 Read block 415 len 4096 18/30 Read block 104 len 12288 19/30 Write block 209 len 40960 data: 8 4 3 3 3 3 4 4 5 6 20/30 Write block 416 len 16384 data: 6 5 6 7 21/30 Write block 515 len 16384 data: 11 11 10 11 22/30 Read block 358 len 36864 23/30 Read block 356 len 12288 24/30 Read block 304 len 20480 25/30 Read block 316 len 28672 26/30 Flush 27/30 Write block 78 len 36864 data: 8 7 10 7 9 11 10 10 12 28/30 Read block 20 len 16384 29/30 Write block 178 len 16384 data: 7 7 7 6 30/30 Write block 504 len 40960 data: 5 6 8 9 6 5 6 7 7 9 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Read 5 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 1 1 27 Done 5 29 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:36.259639822Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:36.259794774Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1515,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 167 167 167 F F F 1 020-039 92 92 92 172 172 172 F F F 2 040-059 92 92 92 172 172 172 F F F 3 060-079 81 81 81 149 149 149 F T T <--- 4 080-099 88 88 88 166 166 166 F T T <--- 5 100-119 93 93 93 174 174 174 F F F 6 120-139 93 93 93 174 174 174 F F F 7 140-159 90 90 90 169 169 169 F F F 8 160-179 93 93 93 174 174 174 F T T <--- 9 180-199 88 88 88 166 166 166 F T T <--- 10 200-219 78 94 94 142 175 175 F F F <--- 11 220-239 91 91 91 171 171 171 F F F 12 240-259 91 94 94 171 175 175 T F F <--- 13 260-279 84 94 94 157 175 175 T F F <--- 14 280-299 86 94 94 161 175 175 F F F <--- 15 300-319 92 94 94 173 175 175 T F F <--- 16 320-339 92 94 94 173 175 175 T F F <--- 17 340-359 92 92 92 173 173 173 F F F 18 360-379 92 92 92 173 173 173 F F F 19 380-399 92 92 92 173 173 173 F F F 20 400-419 93 94 94 174 175 175 F F F <--- 21 420-439 92 92 92 173 173 173 F F F 22 440-459 92 94 94 173 175 175 T F F <--- 23 460-479 92 92 92 173 173 173 F F F 24 480-499 92 92 92 173 173 173 F F F 25 500-519 92 94 94 173 175 175 F T T <--- 26 520-539 90 90 90 169 169 169 F F F 27 540-559 92 94 94 173 175 175 T F F <--- 28 560-579 92 92 92 173 173 173 F F F 29 580-599 92 92 92 173 173 173 F F F Max gen: 94, Max flush: 175 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 167 167 167 F F F 1 020-039 92 92 92 172 172 172 F F F 2 040-059 92 92 92 172 172 172 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 95 95 95 176 176 176 F F F 5 100-119 93 93 93 174 174 174 F F F 6 120-139 93 93 93 174 174 174 F F F 7 140-159 90 90 90 169 169 169 F F F 8 160-179 95 95 95 176 176 176 F F F 9 180-199 95 95 95 176 176 176 F F F 10 200-219 94 94 94 175 175 175 F F F 11 220-239 91 91 91 171 171 171 F F F 12 240-259 94 94 94 175 175 175 F F F 13 260-279 94 94 94 175 175 175 F F F 14 280-299 94 94 94 175 175 175 F F F 15 300-319 94 94 94 175 175 175 F F F 16 320-339 94 94 94 175 175 175 F F F 17 340-359 92 92 92 173 173 173 F F F 18 360-379 92 92 92 173 173 173 F F F 19 380-399 92 92 92 173 173 173 F F F 20 400-419 94 94 94 175 175 175 F F F 21 420-439 92 92 92 173 173 173 F F F 22 440-459 94 94 94 175 175 175 F F F 23 460-479 92 92 92 173 173 173 F F F 24 480-499 92 92 92 173 173 173 F F F 25 500-519 95 95 95 176 176 176 F F F 26 520-539 90 90 90 169 169 169 F F F 27 540-559 94 94 94 175 175 175 F F F 28 560-579 92 92 92 173 173 173 F F F 29 580-599 92 92 92 173 173 173 F F F Max gen: 95, Max flush: 176 Begin loop 48 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.517144098Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.525357709Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.525637816Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.525672432Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.525681961Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.5256937Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523} {"msg":"Crucible 9bb8f078-17aa-41e5-a503-1c2341010a33 has session id: 030dc573-af2c-451f-a9ba-c7ccafa1e58e","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526002214Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"Upstairs opts: Upstairs UUID: 9bb8f078-17aa-41e5-a503-1c2341010a33, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526058637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"Crucible stats registered with UUID: 9bb8f078-17aa-41e5-a503-1c2341010a33","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526071366Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:40.526080525Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526210789Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526226518Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526236626Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.52638077Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526398287Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523} {"msg":"9bb8f078-17aa-41e5-a503-1c2341010a33 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526432803Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526442192Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"client":"0","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526451871Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"client":"1","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.52646145Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"client":"2","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526470999Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526480938Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526490737Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55490)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.52654618Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"io task","client":"0","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43978)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526557169Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"io task","client":"1","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56564)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.526569497Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"io task","client":"2","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527238879Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"client":"0","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527264776Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"client":"1","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527336658Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"client":"2","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527546083Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527555802Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"[0]R flush_numbers[0..12]: [167, 172, 172, 176, 176, 174, 174, 169, 176, 176, 175, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527637863Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"downstairs","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"[0]R generation[0..12]: [89, 92, 92, 95, 95, 93, 93, 90, 95, 95, 94, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527649151Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"downstairs","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.52766018Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"downstairs","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"[1]R flush_numbers[0..12]: [167, 172, 172, 176, 176, 174, 174, 169, 176, 176, 175, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527671169Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"downstairs","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"[1]R generation[0..12]: [89, 92, 92, 95, 95, 93, 93, 90, 95, 95, 94, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527681558Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"downstairs","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527692596Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"downstairs","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"[2]R flush_numbers[0..12]: [167, 172, 172, 176, 176, 174, 174, 169, 176, 176, 175, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527703225Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"downstairs","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"[2]R generation[0..12]: [89, 92, 92, 95, 95, 93, 93, 90, 95, 95, 94, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527713614Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"downstairs","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527726532Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"downstairs","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527736081Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"downstairs","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.52774579Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"downstairs","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527755169Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"downstairs","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527764658Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"":"downstairs","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527774227Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527782996Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"client":"0","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527792605Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"client":"1","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527802274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"client":"2","":"downstairs","session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527812122Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527821191Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"9bb8f078-17aa-41e5-a503-1c2341010a33 is now active with session: 030dc573-af2c-451f-a9ba-c7ccafa1e58e","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.52783109Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527839799Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"session_id":"030dc573-af2c-451f-a9ba-c7ccafa1e58e"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527848828Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523} {"msg":"Activated sub_volume 9bb8f078-17aa-41e5-a503-1c2341010a33","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527856927Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.527865066Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.52817246Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"task":"crutest"} 01/30 Write block 163 len 4096 data: 8 02/30 Read block 165 len 12288 03/30 Read block 242 len 36864 04/30 Write block 567 len 40960 data: 7 8 4 6 9 10 9 10 10 11 05/30 Write block 412 len 24576 data: 7 6 6 6 7 6 06/30 Write block 325 len 28672 data: 7 5 5 8 8 9 7 07/30 Read block 272 len 12288 08/30 Write block 442 len 4096 data: 11 09/30 Flush 10/30 Write block 318 len 12288 data: 11 13 11 11/30 Read block 103 len 28672 12/30 Read block 234 len 12288 13/30 Write block 427 len 32768 data: 9 8 8 8 9 10 11 11 14/30 Write block 382 len 32768 data: 9 10 8 6 7 6 5 4 15/30 Write block 5 len 4096 data: 10 16/30 Read block 566 len 28672 17/30 Write block 484 len 36864 data: 7 6 6 5 9 8 10 10 14 18/30 Read block 475 len 24576 19/30 Flush 20/30 Write block 568 len 4096 data: 9 21/30 Write block 562 len 24576 data: 7 7 9 7 7 8 22/30 Flush 23/30 Write block 151 len 28672 data: 11 13 12 10 10 9 10 24/30 Write block 335 len 28672 data: 11 10 11 9 10 9 6 25/30 Write block 1 len 16384 data: 4 5 6 10 26/30 Read block 236 len 36864 27/30 Read block 319 len 20480 28/30 Read block 147 len 40960 29/30 Read block 301 len 28672 30/30 Read block 349 len 8192 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Write 8 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 26 0 27 Done 29 4 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:40.536881054Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:40.537061393Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1523,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 89 96 178 167 178 T F T <--- 1 020-039 92 92 92 172 172 172 F F F 2 040-059 92 92 92 172 172 172 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 95 95 95 176 176 176 F F F 5 100-119 93 93 93 174 174 174 F F F 6 120-139 93 93 93 174 174 174 F F F 7 140-159 90 90 90 169 169 169 T F T <--- 8 160-179 96 95 96 177 176 177 F T F <--- 9 180-199 95 95 95 176 176 176 F F F 10 200-219 94 94 94 175 175 175 F F F 11 220-239 91 91 91 171 171 171 F F F 12 240-259 94 94 94 175 175 175 F F F 13 260-279 94 94 94 175 175 175 F F F 14 280-299 94 94 94 175 175 175 F F F 15 300-319 96 94 96 178 175 178 F F F <--- 16 320-339 96 94 96 178 175 178 T F T <--- 17 340-359 92 92 92 173 173 173 T F T <--- 18 360-379 92 92 92 173 173 173 F F F 19 380-399 96 92 96 178 173 178 F F F <--- 20 400-419 96 94 96 177 175 177 F F F <--- 21 420-439 96 92 96 178 173 178 F F F <--- 22 440-459 96 94 96 177 175 177 F F F <--- 23 460-479 92 92 92 173 173 173 F F F 24 480-499 96 92 96 178 173 178 F F F <--- 25 500-519 95 95 95 176 176 176 F F F 26 520-539 90 90 90 169 169 169 F F F 27 540-559 94 94 94 175 175 175 F F F 28 560-579 96 92 96 179 173 179 F T F <--- 29 580-599 92 92 92 173 173 173 F F F Max gen: 96, Max flush: 179 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 180 180 180 F F F 1 020-039 92 92 92 172 172 172 F F F 2 040-059 92 92 92 172 172 172 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 95 95 95 176 176 176 F F F 5 100-119 93 93 93 174 174 174 F F F 6 120-139 93 93 93 174 174 174 F F F 7 140-159 97 97 97 180 180 180 F F F 8 160-179 96 96 96 177 177 177 F F F 9 180-199 95 95 95 176 176 176 F F F 10 200-219 94 94 94 175 175 175 F F F 11 220-239 91 91 91 171 171 171 F F F 12 240-259 94 94 94 175 175 175 F F F 13 260-279 94 94 94 175 175 175 F F F 14 280-299 94 94 94 175 175 175 F F F 15 300-319 96 96 96 178 178 178 F F F 16 320-339 97 97 97 180 180 180 F F F 17 340-359 97 97 97 180 180 180 F F F 18 360-379 92 92 92 173 173 173 F F F 19 380-399 96 96 96 178 178 178 F F F 20 400-419 96 96 96 177 177 177 F F F 21 420-439 96 96 96 178 178 178 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 92 92 92 173 173 173 F F F 24 480-499 96 96 96 178 178 178 F F F 25 500-519 95 95 95 176 176 176 F F F 26 520-539 90 90 90 169 169 169 F F F 27 540-559 94 94 94 175 175 175 F F F 28 560-579 96 96 96 179 179 179 F F F 29 580-599 92 92 92 173 173 173 F F F Max gen: 97, Max flush: 180 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.783652373Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.791659308Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.791931606Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.791961583Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.791970152Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.79198161Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531} {"msg":"Crucible 759f25dc-e708-42ea-b4a9-b67833ca4242 has session id: 4f8dc9cc-e773-430f-963c-1359edc67e97","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.792321851Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"Upstairs opts: Upstairs UUID: 759f25dc-e708-42ea-b4a9-b67833ca4242, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.792405881Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"Crucible stats registered with UUID: 759f25dc-e708-42ea-b4a9-b67833ca4242","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.79241639Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:44.792425559Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.79258776Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.792602428Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.792612367Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.792772508Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.792796785Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531} {"msg":"759f25dc-e708-42ea-b4a9-b67833ca4242 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.792821972Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.792831541Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"client":"0","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.79284123Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"client":"1","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.792850899Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"client":"2","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.792860438Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.792870397Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.792880325Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"ds_connection connected from Ok(127.0.0.1:34810)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.792935419Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"io task","client":"2","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"ds_connection connected from Ok(127.0.0.1:58595)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.793047946Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"io task","client":"1","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"ds_connection connected from Ok(127.0.0.1:55362)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.793072743Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"io task","client":"0","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.793627168Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"client":"2","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.793652375Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"client":"0","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.793664034Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"client":"1","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.793922074Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.793934122Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"[0]R flush_numbers[0..12]: [180, 172, 172, 176, 176, 174, 174, 180, 177, 176, 175, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794002414Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"downstairs","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"[0]R generation[0..12]: [97, 92, 92, 95, 95, 93, 93, 97, 96, 95, 94, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794013633Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"downstairs","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794024622Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"downstairs","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"[1]R flush_numbers[0..12]: [180, 172, 172, 176, 176, 174, 174, 180, 177, 176, 175, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794035271Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"downstairs","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"[1]R generation[0..12]: [97, 92, 92, 95, 95, 93, 93, 97, 96, 95, 94, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794045729Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"downstairs","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794056628Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"downstairs","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"[2]R flush_numbers[0..12]: [180, 172, 172, 176, 176, 174, 174, 180, 177, 176, 175, 171]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794067367Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"downstairs","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"[2]R generation[0..12]: [97, 92, 92, 95, 95, 93, 93, 97, 96, 95, 94, 91]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794077666Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"downstairs","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794088574Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"downstairs","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794097993Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"downstairs","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794107652Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"downstairs","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794117091Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"downstairs","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.79412641Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"":"downstairs","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794136549Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794146738Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"client":"0","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794156426Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"client":"1","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794166115Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"client":"2","":"downstairs","session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794175824Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794184903Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"759f25dc-e708-42ea-b4a9-b67833ca4242 is now active with session: 4f8dc9cc-e773-430f-963c-1359edc67e97","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794194982Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794203811Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"session_id":"4f8dc9cc-e773-430f-963c-1359edc67e97"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.79421306Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531} {"msg":"Activated sub_volume 759f25dc-e708-42ea-b4a9-b67833ca4242","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794221129Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794229228Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.794530503Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"task":"crutest"} 01/30 Write block 314 len 36864 data: 4 7 9 11 12 14 12 13 10 02/30 Read block 200 len 40960 03/30 Write block 549 len 16384 data: 14 14 12 12 04/30 Read block 428 len 4096 05/30 Flush 06/30 Write block 293 len 24576 data: 10 12 11 10 11 11 07/30 Read block 241 len 36864 08/30 Write block 203 len 32768 data: 7 8 7 6 5 7 9 5 09/30 Read block 540 len 12288 10/30 Read block 432 len 16384 11/30 Write block 391 len 32768 data: 5 4 5 6 6 6 7 9 12/30 Write block 416 len 32768 data: 8 7 7 8 9 8 6 6 13/30 Write block 505 len 4096 data: 7 14/30 Read block 381 len 40960 15/30 Write block 347 len 8192 data: 5 6 16/30 Read block 590 len 40960 17/30 Flush 18/30 Write block 266 len 24576 data: 7 9 9 10 11 9 19/30 Read block 13 len 4096 20/30 Read block 67 len 20480 21/30 Write block 563 len 32768 data: 8 10 8 8 9 10 5 7 22/30 Read block 27 len 28672 23/30 Read block 338 len 28672 24/30 Read block 84 len 4096 25/30 Flush 26/30 Write block 353 len 32768 data: 11 12 8 6 4 6 6 7 27/30 Write block 254 len 16384 data: 5 4 6 7 28/30 Read block 457 len 12288 29/30 Write block 234 len 28672 data: 6 10 8 9 12 13 11 30/30 Read block 112 len 8192 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:44.804298393Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:44.804477542Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1531,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 180 180 180 F F F 1 020-039 92 92 92 172 172 172 F F F 2 040-059 92 92 92 172 172 172 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 95 95 95 176 176 176 F F F 5 100-119 93 93 93 174 174 174 F F F 6 120-139 93 93 93 174 174 174 F F F 7 140-159 97 97 97 180 180 180 F F F 8 160-179 96 96 96 177 177 177 F F F 9 180-199 95 95 95 176 176 176 F F F 10 200-219 98 94 98 182 175 182 F F F <--- 11 220-239 91 91 91 171 171 171 T F T <--- 12 240-259 94 94 94 175 175 175 T F T <--- 13 260-279 98 94 98 183 175 183 F F F <--- 14 280-299 98 94 98 182 175 182 F F F <--- 15 300-319 98 96 98 181 178 181 F T F <--- 16 320-339 98 97 98 181 180 181 F T F <--- 17 340-359 98 97 98 182 180 182 T F T <--- 18 360-379 92 92 92 173 173 173 T F T <--- 19 380-399 98 96 98 182 178 182 F F F <--- 20 400-419 98 96 98 182 177 182 F F F <--- 21 420-439 98 96 98 182 178 182 F F F <--- 22 440-459 96 96 96 177 177 177 F F F 23 460-479 92 92 92 173 173 173 F F F 24 480-499 96 96 96 178 178 178 F F F 25 500-519 98 95 98 182 176 182 F F F <--- 26 520-539 90 90 90 169 169 169 F F F 27 540-559 98 94 98 181 175 181 F F F <--- 28 560-579 98 96 98 183 179 183 F F F <--- 29 580-599 92 92 92 173 173 173 F F F Max gen: 98, Max flush: 183 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 97 97 97 180 180 180 F F F 1 020-039 92 92 92 172 172 172 F F F 2 040-059 92 92 92 172 172 172 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 95 95 95 176 176 176 F F F 5 100-119 93 93 93 174 174 174 F F F 6 120-139 93 93 93 174 174 174 F F F 7 140-159 97 97 97 180 180 180 F F F 8 160-179 96 96 96 177 177 177 F F F 9 180-199 95 95 95 176 176 176 F F F 10 200-219 98 98 98 182 182 182 F F F 11 220-239 99 99 99 184 184 184 F F F 12 240-259 99 99 99 184 184 184 F F F 13 260-279 98 98 98 183 183 183 F F F 14 280-299 98 98 98 182 182 182 F F F 15 300-319 98 98 98 181 181 181 F F F 16 320-339 98 98 98 181 181 181 F F F 17 340-359 99 99 99 184 184 184 F F F 18 360-379 99 99 99 184 184 184 F F F 19 380-399 98 98 98 182 182 182 F F F 20 400-419 98 98 98 182 182 182 F F F 21 420-439 98 98 98 182 182 182 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 92 92 92 173 173 173 F F F 24 480-499 96 96 96 178 178 178 F F F 25 500-519 98 98 98 182 182 182 F F F 26 520-539 90 90 90 169 169 169 F F F 27 540-559 98 98 98 181 181 181 F F F 28 560-579 98 98 98 183 183 183 F F F 29 580-599 92 92 92 173 173 173 F F F Max gen: 99, Max flush: 184 Begin loop 50 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.058466427Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.066527926Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.066812193Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.06683908Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.066850459Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.066862137Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539} {"msg":"Crucible f22865a3-25f7-4c88-89e5-c1d1b9d47e17 has session id: 59eca515-8f87-4524-af61-34c06364a1ad","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067164842Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"Upstairs opts: Upstairs UUID: f22865a3-25f7-4c88-89e5-c1d1b9d47e17, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067195119Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"Crucible stats registered with UUID: f22865a3-25f7-4c88-89e5-c1d1b9d47e17","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067206107Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:49.067215256Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067342871Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067373938Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067384956Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.06752616Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067544978Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539} {"msg":"f22865a3-25f7-4c88-89e5-c1d1b9d47e17 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067569765Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067579304Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"client":"0","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067589013Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"client":"1","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067598611Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"client":"2","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.06760811Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067619759Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067629648Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:61592)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067649436Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"io task","client":"2","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:39506)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067685561Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"io task","client":"0","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"ds_connection connected from Ok(127.0.0.1:37163)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.067751274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"io task","client":"1","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068248706Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"client":"0","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.06829351Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"client":"1","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068448172Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"client":"2","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068541581Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068558799Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"[0]R flush_numbers[0..12]: [180, 172, 172, 176, 176, 174, 174, 180, 177, 176, 182, 184]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068711732Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"downstairs","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"[0]R generation[0..12]: [97, 92, 92, 95, 95, 93, 93, 97, 96, 95, 98, 99]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.06872311Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"downstairs","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068734319Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"downstairs","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"[1]R flush_numbers[0..12]: [180, 172, 172, 176, 176, 174, 174, 180, 177, 176, 182, 184]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068745118Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"downstairs","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"[1]R generation[0..12]: [97, 92, 92, 95, 95, 93, 93, 97, 96, 95, 98, 99]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068757526Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"downstairs","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068768495Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"downstairs","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"[2]R flush_numbers[0..12]: [180, 172, 172, 176, 176, 174, 174, 180, 177, 176, 182, 184]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068779264Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"downstairs","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"[2]R generation[0..12]: [97, 92, 92, 95, 95, 93, 93, 97, 96, 95, 98, 99]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068789572Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"downstairs","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068800511Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"downstairs","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.06880987Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"downstairs","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068819579Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"downstairs","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068829018Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"downstairs","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068838407Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"":"downstairs","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068848306Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068857305Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"client":"0","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068866963Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"client":"1","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068876602Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"client":"2","":"downstairs","session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068886591Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.06889566Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"f22865a3-25f7-4c88-89e5-c1d1b9d47e17 is now active with session: 59eca515-8f87-4524-af61-34c06364a1ad","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068906719Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068915488Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"session_id":"59eca515-8f87-4524-af61-34c06364a1ad"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068924597Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539} {"msg":"Activated sub_volume f22865a3-25f7-4c88-89e5-c1d1b9d47e17","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068932616Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.068940685Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.069250169Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"task":"crutest"} 01/30 Write block 325 len 12288 data: 8 6 6 02/30 Read block 291 len 32768 03/30 Write block 22 len 40960 data: 8 6 7 8 12 13 15 15 16 17 04/30 Read block 237 len 12288 05/30 Read block 587 len 32768 06/30 Write block 235 len 28672 data: 11 9 10 13 14 12 13 07/30 Write block 220 len 12288 data: 5 5 7 08/30 Read block 106 len 40960 09/30 Read block 201 len 24576 10/30 Read block 193 len 36864 11/30 Write block 165 len 20480 data: 9 9 7 8 7 12/30 Write block 6 len 12288 data: 11 10 11 13/30 Read block 444 len 28672 14/30 Read block 534 len 40960 15/30 Write block 282 len 16384 data: 8 6 6 6 16/30 Read block 147 len 24576 17/30 Read block 406 len 8192 18/30 Flush 19/30 Write block 577 len 8192 data: 12 8 20/30 Write block 324 len 12288 data: 8 9 7 21/30 Read block 295 len 16384 22/30 Read block 399 len 40960 23/30 Read block 56 len 4096 24/30 Read block 471 len 12288 25/30 Write block 481 len 16384 data: 6 5 5 8 26/30 Read block 518 len 12288 27/30 Read block 143 len 4096 28/30 Read block 313 len 12288 29/30 Write block 421 len 8192 data: 9 7 30/30 Read block 16 len 32768 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Read 8 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Write 3 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Write 2 Done Done Sent false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 26 26 Done 30 30 4 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:49.078475602Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:49.07865997Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1539,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 97 185 185 180 F F F <--- 1 020-039 100 100 92 185 185 172 F F T <--- 2 040-059 92 92 92 172 172 172 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 95 95 95 176 176 176 F F F 5 100-119 93 93 93 174 174 174 F F F 6 120-139 93 93 93 174 174 174 F F F 7 140-159 97 97 97 180 180 180 F F F 8 160-179 100 100 96 185 185 177 F F F <--- 9 180-199 95 95 95 176 176 176 F F F 10 200-219 98 98 98 182 182 182 F F F 11 220-239 100 100 99 185 185 184 F F F <--- 12 240-259 100 100 99 185 185 184 F F F <--- 13 260-279 98 98 98 183 183 183 F F F 14 280-299 100 100 98 185 185 182 F F F <--- 15 300-319 98 98 98 181 181 181 F F F 16 320-339 100 100 98 185 185 181 T T T <--- 17 340-359 99 99 99 184 184 184 F F F 18 360-379 99 99 99 184 184 184 F F F 19 380-399 98 98 98 182 182 182 F F F 20 400-419 98 98 98 182 182 182 F F F 21 420-439 98 98 98 182 182 182 T T F <--- 22 440-459 96 96 96 177 177 177 F F F 23 460-479 92 92 92 173 173 173 F F F 24 480-499 96 96 96 178 178 178 T T F <--- 25 500-519 98 98 98 182 182 182 F F F 26 520-539 90 90 90 169 169 169 F F F 27 540-559 98 98 98 181 181 181 F F F 28 560-579 98 98 98 183 183 183 T T F <--- 29 580-599 92 92 92 173 173 173 F F F Max gen: 100, Max flush: 185 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 185 185 185 F F F 1 020-039 100 100 100 185 185 185 F F F 2 040-059 92 92 92 172 172 172 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 95 95 95 176 176 176 F F F 5 100-119 93 93 93 174 174 174 F F F 6 120-139 93 93 93 174 174 174 F F F 7 140-159 97 97 97 180 180 180 F F F 8 160-179 100 100 100 185 185 185 F F F 9 180-199 95 95 95 176 176 176 F F F 10 200-219 98 98 98 182 182 182 F F F 11 220-239 100 100 100 185 185 185 F F F 12 240-259 100 100 100 185 185 185 F F F 13 260-279 98 98 98 183 183 183 F F F 14 280-299 100 100 100 185 185 185 F F F 15 300-319 98 98 98 181 181 181 F F F 16 320-339 101 101 101 186 186 186 F F F 17 340-359 99 99 99 184 184 184 F F F 18 360-379 99 99 99 184 184 184 F F F 19 380-399 98 98 98 182 182 182 F F F 20 400-419 98 98 98 182 182 182 F F F 21 420-439 101 101 101 186 186 186 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 92 92 92 173 173 173 F F F 24 480-499 101 101 101 186 186 186 F F F 25 500-519 98 98 98 182 182 182 F F F 26 520-539 90 90 90 169 169 169 F F F 27 540-559 98 98 98 181 181 181 F F F 28 560-579 101 101 101 186 186 186 F F F 29 580-599 92 92 92 173 173 173 F F F Max gen: 101, Max flush: 186 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.322790787Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.330946745Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.331224033Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.331257169Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.331266558Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.331278147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547} {"msg":"Crucible 517e8aae-2a33-4115-b3eb-bcd02a827e92 has session id: dd21d224-59a9-41db-ac85-52af830c91e8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.331615537Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"Upstairs opts: Upstairs UUID: 517e8aae-2a33-4115-b3eb-bcd02a827e92, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.331645024Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"Crucible stats registered with UUID: 517e8aae-2a33-4115-b3eb-bcd02a827e92","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.331656402Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:53.331668541Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.331792956Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.331808285Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.331818463Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.331993243Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.332012031Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547} {"msg":"517e8aae-2a33-4115-b3eb-bcd02a827e92 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.332029579Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.332038308Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"client":"0","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.332048077Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"client":"1","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.332057516Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"client":"2","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.332067114Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.332079593Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.3321031Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:59541)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.332144305Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"io task","client":"2","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:60647)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.332174012Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"io task","client":"0","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:47993)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.332252623Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"io task","client":"1","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.332745015Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"client":"0","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.332830635Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"client":"2","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.332859572Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"client":"1","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333053609Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333139499Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"[0]R flush_numbers[0..12]: [185, 185, 172, 176, 176, 174, 174, 180, 185, 176, 182, 185]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333213851Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"downstairs","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"[0]R generation[0..12]: [100, 100, 92, 95, 95, 93, 93, 97, 100, 95, 98, 100]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333225429Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"downstairs","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333236538Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"downstairs","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"[1]R flush_numbers[0..12]: [185, 185, 172, 176, 176, 174, 174, 180, 185, 176, 182, 185]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333247267Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"downstairs","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"[1]R generation[0..12]: [100, 100, 92, 95, 95, 93, 93, 97, 100, 95, 98, 100]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333257785Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"downstairs","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333268834Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"downstairs","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"[2]R flush_numbers[0..12]: [185, 185, 172, 176, 176, 174, 174, 180, 185, 176, 182, 185]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333279483Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"downstairs","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"[2]R generation[0..12]: [100, 100, 92, 95, 95, 93, 93, 97, 100, 95, 98, 100]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333289812Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"downstairs","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.3333007Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"downstairs","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333312789Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"downstairs","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333322728Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"downstairs","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333332017Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"downstairs","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333341336Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"":"downstairs","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333351065Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333359794Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"client":"0","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333369402Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"client":"1","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333379141Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"client":"2","":"downstairs","session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.33338897Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333398359Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"517e8aae-2a33-4115-b3eb-bcd02a827e92 is now active with session: dd21d224-59a9-41db-ac85-52af830c91e8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333408288Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333417227Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"session_id":"dd21d224-59a9-41db-ac85-52af830c91e8"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333426456Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547} {"msg":"Activated sub_volume 517e8aae-2a33-4115-b3eb-bcd02a827e92","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333434425Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333442514Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.333766806Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"task":"crutest"} 01/30 Flush 02/30 Write block 241 len 24576 data: 14 14 12 13 11 11 03/30 Read block 36 len 32768 04/30 Write block 40 len 40960 data: 8 7 6 5 7 8 9 9 7 8 05/30 Read block 396 len 28672 06/30 Write block 270 len 28672 data: 12 10 11 11 11 13 13 07/30 Read block 149 len 36864 08/30 Write block 14 len 24576 data: 7 6 6 6 5 7 09/30 Write block 29 len 36864 data: 16 17 18 17 15 12 10 8 6 10/30 Write block 544 len 32768 data: 15 13 15 16 15 15 15 13 11/30 Read block 285 len 12288 12/30 Read block 436 len 8192 13/30 Write block 139 len 40960 data: 5 6 6 7 6 8 9 9 10 10 14/30 Read block 112 len 4096 15/30 Read block 246 len 20480 16/30 Read block 314 len 32768 17/30 Flush 18/30 Write block 234 len 4096 data: 7 19/30 Write block 304 len 28672 data: 10 14 13 12 9 8 7 20/30 Write block 142 len 32768 data: 8 7 9 10 10 11 11 9 21/30 Read block 68 len 16384 22/30 Write block 253 len 20480 data: 7 6 5 7 8 23/30 Read block 395 len 28672 24/30 Write block 139 len 20480 data: 6 7 7 9 8 25/30 Read block 127 len 20480 26/30 Read block 563 len 16384 27/30 Write block 233 len 4096 data: 7 28/30 Write block 58 len 8192 data: 8 6 29/30 Read block 263 len 12288 30/30 Read block 501 len 28672 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 6 Done Done Done false 1002 Acked Read 8 Done Sent Done false 1003 Acked Write 10 Done Sent Done false 1004 Acked Read 7 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 29 1 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1000)) Some(JobId(1016)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:53.342926527Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:53.343135542Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1547,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 100 102 188 185 188 F F F <--- 1 020-039 102 100 102 188 185 188 F F F <--- 2 040-059 102 92 102 188 172 188 T F T <--- 3 060-079 95 95 95 176 176 176 F F F 4 080-099 95 95 95 176 176 176 F F F 5 100-119 93 93 93 174 174 174 F F F 6 120-139 102 93 102 188 174 188 T F T <--- 7 140-159 102 97 102 188 180 188 T F T <--- 8 160-179 100 100 100 185 185 185 F F F 9 180-199 95 95 95 176 176 176 F F F 10 200-219 98 98 98 182 182 182 F F F 11 220-239 100 100 100 185 185 185 T F T <--- 12 240-259 102 100 102 188 185 188 T T T <--- 13 260-279 102 98 102 188 183 188 F F F <--- 14 280-299 100 100 100 185 185 185 F F F 15 300-319 98 98 98 181 181 181 T F T <--- 16 320-339 101 101 101 186 186 186 F F F 17 340-359 99 99 99 184 184 184 F F F 18 360-379 99 99 99 184 184 184 F F F 19 380-399 98 98 98 182 182 182 F F F 20 400-419 98 98 98 182 182 182 F F F 21 420-439 101 101 101 186 186 186 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 92 92 92 173 173 173 F F F 24 480-499 101 101 101 186 186 186 F F F 25 500-519 98 98 98 182 182 182 F F F 26 520-539 90 90 90 169 169 169 F F F 27 540-559 102 98 102 188 181 188 F F F <--- 28 560-579 101 101 101 186 186 186 F F F 29 580-599 92 92 92 173 173 173 F F F Max gen: 102, Max flush: 188 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 188 188 188 F F F 1 020-039 102 102 102 188 188 188 F F F 2 040-059 103 103 103 189 189 189 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 95 95 95 176 176 176 F F F 5 100-119 93 93 93 174 174 174 F F F 6 120-139 103 103 103 189 189 189 F F F 7 140-159 103 103 103 189 189 189 F F F 8 160-179 100 100 100 185 185 185 F F F 9 180-199 95 95 95 176 176 176 F F F 10 200-219 98 98 98 182 182 182 F F F 11 220-239 103 103 103 189 189 189 F F F 12 240-259 103 103 103 189 189 189 F F F 13 260-279 102 102 102 188 188 188 F F F 14 280-299 100 100 100 185 185 185 F F F 15 300-319 103 103 103 189 189 189 F F F 16 320-339 101 101 101 186 186 186 F F F 17 340-359 99 99 99 184 184 184 F F F 18 360-379 99 99 99 184 184 184 F F F 19 380-399 98 98 98 182 182 182 F F F 20 400-419 98 98 98 182 182 182 F F F 21 420-439 101 101 101 186 186 186 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 92 92 92 173 173 173 F F F 24 480-499 101 101 101 186 186 186 F F F 25 500-519 98 98 98 182 182 182 F F F 26 520-539 90 90 90 169 169 169 F F F 27 540-559 102 102 102 188 188 188 F F F 28 560-579 101 101 101 186 186 186 F F F 29 580-599 92 92 92 173 173 173 F F F Max gen: 103, Max flush: 189 Begin loop 52 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.587309124Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:57.594474958Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.594527002Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.601551882Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.601800583Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.60182908Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.601837789Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.601849287Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555} {"msg":"Crucible f4dfa11f-1db6-456d-8a5b-e43fa8cfbff3 has session id: c9430da2-8f75-4b1a-808f-ad6b31ea2067","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602152312Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"Upstairs opts: Upstairs UUID: f4dfa11f-1db6-456d-8a5b-e43fa8cfbff3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602174389Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"Crucible stats registered with UUID: f4dfa11f-1db6-456d-8a5b-e43fa8cfbff3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602184498Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:57.602193507Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.60233854Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602367547Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602380955Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.60250875Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602519679Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555} {"msg":"f4dfa11f-1db6-456d-8a5b-e43fa8cfbff3 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602535737Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602544656Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"client":"0","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602554375Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"client":"1","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602564094Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"client":"2","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602573563Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602583402Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.60259343Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"ds_connection connected from Ok(127.0.0.1:63414)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602647204Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"io task","client":"0","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"ds_connection connected from Ok(127.0.0.1:63894)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602658013Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"io task","client":"2","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"ds_connection connected from Ok(127.0.0.1:53545)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.602668412Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"io task","client":"1","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603295548Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"client":"2","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603321155Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"client":"1","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603332914Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"client":"0","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603589384Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603599373Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"[0]R flush_numbers[0..12]: [188, 188, 189, 176, 176, 174, 189, 189, 185, 176, 182, 189]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603617531Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"downstairs","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"[0]R generation[0..12]: [102, 102, 103, 95, 95, 93, 103, 103, 100, 95, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.60362822Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"downstairs","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603639408Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"downstairs","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"[1]R flush_numbers[0..12]: [188, 188, 189, 176, 176, 174, 189, 189, 185, 176, 182, 189]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603650167Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"downstairs","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"[1]R generation[0..12]: [102, 102, 103, 95, 95, 93, 103, 103, 100, 95, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603660666Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"downstairs","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603671484Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"downstairs","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"[2]R flush_numbers[0..12]: [188, 188, 189, 176, 176, 174, 189, 189, 185, 176, 182, 189]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603681863Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"downstairs","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"[2]R generation[0..12]: [102, 102, 103, 95, 95, 93, 103, 103, 100, 95, 98, 103]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603692582Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"downstairs","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603703351Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"downstairs","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.60371277Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"downstairs","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603722399Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"downstairs","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603731577Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"downstairs","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603742756Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"":"downstairs","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603752505Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603761114Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"client":"0","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603770673Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"client":"1","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603780152Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"client":"2","":"downstairs","session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603789901Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.60379898Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"f4dfa11f-1db6-456d-8a5b-e43fa8cfbff3 is now active with session: c9430da2-8f75-4b1a-808f-ad6b31ea2067","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603808798Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"session_id":"c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603817587Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 c9430da2-8f75-4b1a-808f-ad6b31ea2067"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603831506Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555} {"msg":"Activated sub_volume f4dfa11f-1db6-456d-8a5b-e43fa8cfbff3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603839635Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.603847634Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.604075157Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"task":"crutest"} 01/30 Write block 236 len 8192 data: 10 11 02/30 Read block 268 len 20480 03/30 Write block 521 len 12288 data: 6 6 6 04/30 Read block 354 len 16384 05/30 Write block 355 len 40960 data: 9 7 5 7 7 8 7 6 6 6 06/30 Read block 234 len 12288 07/30 Read block 354 len 36864 08/30 Read block 566 len 32768 09/30 Flush 10/30 Write block 517 len 12288 data: 11 12 10 11/30 Read block 537 len 32768 12/30 Write block 211 len 12288 data: 4 4 4 13/30 Read block 103 len 4096 14/30 Write block 307 len 16384 data: 13 10 9 8 15/30 Read block 467 len 32768 16/30 Read block 466 len 4096 17/30 Flush 18/30 Write block 253 len 28672 data: 8 7 6 8 9 4 4 19/30 Flush 20/30 Write block 177 len 40960 data: 6 8 8 8 7 5 6 4 5 4 21/30 Read block 266 len 28672 22/30 Read block 212 len 20480 23/30 Read block 168 len 32768 24/30 Write block 184 len 12288 data: 5 6 5 25/30 Read block 326 len 24576 26/30 Write block 569 len 28672 data: 6 8 10 11 10 11 11 27/30 Write block 128 len 32768 data: 10 10 10 8 8 7 6 7 28/30 Write block 116 len 12288 data: 9 10 12 29/30 Read block 182 len 24576 30/30 Read block 269 len 20480 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 3 Done Done Done false 1003 Acked Read 4 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Read 9 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 0 28 Done 3 29 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:42:57.61321983Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:42:57.613395869Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1555,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 188 188 188 F F F 1 020-039 102 102 102 188 188 188 F F F 2 040-059 103 103 103 189 189 189 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 95 95 95 176 176 176 F F F 5 100-119 93 93 93 174 174 174 F T T <--- 6 120-139 103 103 103 189 189 189 F T T <--- 7 140-159 103 103 103 189 189 189 F F F 8 160-179 100 100 100 185 185 185 F T T <--- 9 180-199 95 95 95 176 176 176 F T T <--- 10 200-219 98 104 104 182 191 191 F F F <--- 11 220-239 103 104 104 189 190 190 T F F <--- 12 240-259 103 104 104 189 192 192 F F F <--- 13 260-279 102 102 102 188 188 188 F F F 14 280-299 100 100 100 185 185 185 F F F 15 300-319 103 104 104 189 191 191 F F F <--- 16 320-339 101 101 101 186 186 186 F F F 17 340-359 99 104 104 184 190 190 F F F <--- 18 360-379 99 104 104 184 190 190 F F F <--- 19 380-399 98 98 98 182 182 182 F F F 20 400-419 98 98 98 182 182 182 F F F 21 420-439 101 101 101 186 186 186 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 92 92 92 173 173 173 F F F 24 480-499 101 101 101 186 186 186 F F F 25 500-519 98 104 104 182 191 191 F F F <--- 26 520-539 90 104 104 169 190 190 T F F <--- 27 540-559 102 102 102 188 188 188 F F F 28 560-579 101 101 101 186 186 186 F T T <--- 29 580-599 92 92 92 173 173 173 F F F Max gen: 104, Max flush: 192 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 188 188 188 F F F 1 020-039 102 102 102 188 188 188 F F F 2 040-059 103 103 103 189 189 189 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 95 95 95 176 176 176 F F F 5 100-119 105 105 105 193 193 193 F F F 6 120-139 105 105 105 193 193 193 F F F 7 140-159 103 103 103 189 189 189 F F F 8 160-179 105 105 105 193 193 193 F F F 9 180-199 105 105 105 193 193 193 F F F 10 200-219 104 104 104 191 191 191 F F F 11 220-239 104 104 104 190 190 190 F F F 12 240-259 104 104 104 192 192 192 F F F 13 260-279 102 102 102 188 188 188 F F F 14 280-299 100 100 100 185 185 185 F F F 15 300-319 104 104 104 191 191 191 F F F 16 320-339 101 101 101 186 186 186 F F F 17 340-359 104 104 104 190 190 190 F F F 18 360-379 104 104 104 190 190 190 F F F 19 380-399 98 98 98 182 182 182 F F F 20 400-419 98 98 98 182 182 182 F F F 21 420-439 101 101 101 186 186 186 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 92 92 92 173 173 173 F F F 24 480-499 101 101 101 186 186 186 F F F 25 500-519 104 104 104 191 191 191 F F F 26 520-539 104 104 104 190 190 190 F F F 27 540-559 102 102 102 188 188 188 F F F 28 560-579 105 105 105 193 193 193 F F F 29 580-599 92 92 92 173 173 173 F F F Max gen: 105, Max flush: 193 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.87265666Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.880921245Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.881202262Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.881227459Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.881239348Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.881251127Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563} {"msg":"Crucible 321cbcfc-fc33-4c82-bff9-4e52cfd9846a has session id: 45a2102c-a80e-41f2-a646-cd9a6a38a1f7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.881590227Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"Upstairs opts: Upstairs UUID: 321cbcfc-fc33-4c82-bff9-4e52cfd9846a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.881619154Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"Crucible stats registered with UUID: 321cbcfc-fc33-4c82-bff9-4e52cfd9846a","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.881630262Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:01.881639651Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.881772706Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.881787914Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.881797973Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.881943116Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.881958994Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563} {"msg":"321cbcfc-fc33-4c82-bff9-4e52cfd9846a active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.88199526Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.882005139Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"client":"0","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.882015007Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"client":"1","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.882024636Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"client":"2","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.882036495Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.882046574Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.882056573Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:41219)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.882108506Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"io task","client":"2","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:63050)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.882138373Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"io task","client":"1","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:59136)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.882218254Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"io task","client":"0","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.882881466Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"client":"2","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.882907773Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"client":"0","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.882959007Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"client":"1","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883231915Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883254753Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"[0]R flush_numbers[0..12]: [188, 188, 189, 176, 176, 193, 193, 189, 193, 193, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883265431Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"downstairs","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"[0]R generation[0..12]: [102, 102, 103, 95, 95, 105, 105, 103, 105, 105, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.8832765Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"downstairs","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883288019Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"downstairs","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"[1]R flush_numbers[0..12]: [188, 188, 189, 176, 176, 193, 193, 189, 193, 193, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883301337Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"downstairs","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"[1]R generation[0..12]: [102, 102, 103, 95, 95, 105, 105, 103, 105, 105, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883312026Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"downstairs","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883322875Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"downstairs","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"[2]R flush_numbers[0..12]: [188, 188, 189, 176, 176, 193, 193, 189, 193, 193, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883333493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"downstairs","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"[2]R generation[0..12]: [102, 102, 103, 95, 95, 105, 105, 103, 105, 105, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883343992Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"downstairs","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883354901Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"downstairs","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.88336421Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"downstairs","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883374179Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"downstairs","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883383568Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"downstairs","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883392907Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"":"downstairs","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883402655Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883411534Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"client":"0","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883421363Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"client":"1","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883431082Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"client":"2","":"downstairs","session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883440721Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.88345113Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"321cbcfc-fc33-4c82-bff9-4e52cfd9846a is now active with session: 45a2102c-a80e-41f2-a646-cd9a6a38a1f7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883461049Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883469718Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"session_id":"45a2102c-a80e-41f2-a646-cd9a6a38a1f7"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883478917Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563} {"msg":"Activated sub_volume 321cbcfc-fc33-4c82-bff9-4e52cfd9846a","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883486956Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883494965Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.883831145Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"task":"crutest"} 01/30 Flush 02/30 Write block 178 len 28672 data: 9 9 9 8 6 7 6 03/30 Write block 550 len 24576 data: 16 14 13 14 11 10 04/30 Write block 90 len 24576 data: 12 10 11 11 11 10 05/30 Read block 89 len 36864 06/30 Read block 590 len 40960 07/30 Read block 514 len 4096 08/30 Read block 306 len 40960 09/30 Write block 19 len 24576 data: 8 8 7 9 7 8 10/30 Read block 425 len 36864 11/30 Write block 18 len 28672 data: 6 9 9 8 10 8 9 12/30 Write block 554 len 24576 data: 12 11 9 7 7 7 13/30 Read block 331 len 4096 14/30 Flush 15/30 Flush 16/30 Write block 93 len 8192 data: 12 12 17/30 Read block 245 len 4096 18/30 Write block 9 len 36864 data: 9 10 9 10 10 8 7 7 7 19/30 Read block 546 len 4096 20/30 Read block 65 len 20480 21/30 Write block 147 len 40960 data: 12 12 10 11 12 14 13 11 11 10 22/30 Read block 241 len 32768 23/30 Write block 351 len 20480 data: 9 11 12 13 10 24/30 Flush 25/30 Write block 310 len 16384 data: 9 4 4 4 26/30 Write block 95 len 8192 data: 11 9 27/30 Write block 148 len 16384 data: 13 11 12 13 28/30 Read block 73 len 28672 29/30 Read block 378 len 32768 30/30 Read block 305 len 24576 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 7 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 8 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 25 26 Done 29 28 4 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:01.892669914Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:01.892853832Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1563,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 102 197 197 188 F F F <--- 1 020-039 106 106 102 195 195 188 F F F <--- 2 040-059 103 103 103 189 189 189 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 106 106 95 197 197 176 T T T <--- 5 100-119 105 105 105 193 193 193 F F F 6 120-139 105 105 105 193 193 193 F F F 7 140-159 106 106 103 197 197 189 T T F <--- 8 160-179 106 106 105 195 195 193 F F T <--- 9 180-199 106 106 105 195 195 193 F F T <--- 10 200-219 104 104 104 191 191 191 F F F 11 220-239 104 104 104 190 190 190 F F F 12 240-259 104 104 104 192 192 192 F F F 13 260-279 102 102 102 188 188 188 F F F 14 280-299 100 100 100 185 185 185 F F F 15 300-319 104 104 104 191 191 191 T T F <--- 16 320-339 101 101 101 186 186 186 F F F 17 340-359 106 106 104 197 197 190 F F F <--- 18 360-379 104 104 104 190 190 190 F F F 19 380-399 98 98 98 182 182 182 F F F 20 400-419 98 98 98 182 182 182 F F F 21 420-439 101 101 101 186 186 186 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 92 92 92 173 173 173 F F F 24 480-499 101 101 101 186 186 186 F F F 25 500-519 104 104 104 191 191 191 F F F 26 520-539 104 104 104 190 190 190 F F F 27 540-559 106 106 102 195 195 188 F F T <--- 28 560-579 105 105 105 193 193 193 F F F 29 580-599 92 92 92 173 173 173 F F F Max gen: 106, Max flush: 197 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 197 197 197 F F F 1 020-039 106 106 106 195 195 195 F F F 2 040-059 103 103 103 189 189 189 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 107 107 107 198 198 198 F F F 5 100-119 105 105 105 193 193 193 F F F 6 120-139 105 105 105 193 193 193 F F F 7 140-159 107 107 107 198 198 198 F F F 8 160-179 106 106 106 195 195 195 F F F 9 180-199 106 106 106 195 195 195 F F F 10 200-219 104 104 104 191 191 191 F F F 11 220-239 104 104 104 190 190 190 F F F 12 240-259 104 104 104 192 192 192 F F F 13 260-279 102 102 102 188 188 188 F F F 14 280-299 100 100 100 185 185 185 F F F 15 300-319 107 107 107 198 198 198 F F F 16 320-339 101 101 101 186 186 186 F F F 17 340-359 106 106 106 197 197 197 F F F 18 360-379 104 104 104 190 190 190 F F F 19 380-399 98 98 98 182 182 182 F F F 20 400-419 98 98 98 182 182 182 F F F 21 420-439 101 101 101 186 186 186 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 92 92 92 173 173 173 F F F 24 480-499 101 101 101 186 186 186 F F F 25 500-519 104 104 104 191 191 191 F F F 26 520-539 104 104 104 190 190 190 F F F 27 540-559 106 106 106 195 195 195 F F F 28 560-579 105 105 105 193 193 193 F F F 29 580-599 92 92 92 173 173 173 F F F Max gen: 107, Max flush: 198 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.134609596Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.142757975Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143038493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.14306434Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143073128Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143084807Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571} {"msg":"Crucible ef575e74-ed2f-43ff-a6e3-e8bf2be3a8cf has session id: a713c269-4616-4e60-b62a-5116cb17d884","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.14348497Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"Upstairs opts: Upstairs UUID: ef575e74-ed2f-43ff-a6e3-e8bf2be3a8cf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143515697Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"Crucible stats registered with UUID: ef575e74-ed2f-43ff-a6e3-e8bf2be3a8cf","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143529265Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:06.143538484Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143669689Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143698505Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143709444Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143836229Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143854747Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571} {"msg":"ef575e74-ed2f-43ff-a6e3-e8bf2be3a8cf active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143878754Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143888183Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"client":"0","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143898342Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"client":"1","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143907991Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"client":"2","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.14391765Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143927569Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143941677Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"ds_connection connected from Ok(127.0.0.1:57749)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.143988752Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"io task","client":"0","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"ds_connection connected from Ok(127.0.0.1:40611)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.144009359Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"io task","client":"2","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"ds_connection connected from Ok(127.0.0.1:57432)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.144021698Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"io task","client":"1","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.144647795Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"client":"0","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.144673692Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"client":"1","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.14468531Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"client":"2","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.14494544Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.144980446Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"[0]R flush_numbers[0..12]: [197, 195, 189, 176, 198, 193, 193, 198, 195, 195, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145046238Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"downstairs","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"[0]R generation[0..12]: [106, 106, 103, 95, 107, 105, 105, 107, 106, 106, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145057587Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"downstairs","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145068726Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"downstairs","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"[1]R flush_numbers[0..12]: [197, 195, 189, 176, 198, 193, 193, 198, 195, 195, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145079234Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"downstairs","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"[1]R generation[0..12]: [106, 106, 103, 95, 107, 105, 105, 107, 106, 106, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145089823Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"downstairs","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145100672Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"downstairs","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"[2]R flush_numbers[0..12]: [197, 195, 189, 176, 198, 193, 193, 198, 195, 195, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145111351Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"downstairs","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"[2]R generation[0..12]: [106, 106, 103, 95, 107, 105, 105, 107, 106, 106, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145121899Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"downstairs","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145134718Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"downstairs","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145144257Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"downstairs","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145153846Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"downstairs","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145163115Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"downstairs","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145172463Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"":"downstairs","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145182122Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145190941Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"client":"0","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.14520042Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"client":"1","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145210129Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"client":"2","":"downstairs","session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145219848Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145228967Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"ef575e74-ed2f-43ff-a6e3-e8bf2be3a8cf is now active with session: a713c269-4616-4e60-b62a-5116cb17d884","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145238916Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145247505Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"session_id":"a713c269-4616-4e60-b62a-5116cb17d884"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145256674Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571} {"msg":"Activated sub_volume ef575e74-ed2f-43ff-a6e3-e8bf2be3a8cf","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145264593Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145272692Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.145574586Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"task":"crutest"} 01/30 Write block 511 len 8192 data: 8 8 02/30 Write block 555 len 40960 data: 12 10 8 8 8 5 7 8 9 11 03/30 Write block 499 len 32768 data: 7 8 7 4 5 6 8 9 04/30 Read block 574 len 36864 05/30 Flush 06/30 Write block 590 len 16384 data: 8 6 7 7 07/30 Read block 345 len 16384 08/30 Read block 175 len 20480 09/30 Flush 10/30 Write block 45 len 28672 data: 9 10 10 8 9 7 12 11/30 Write block 494 len 36864 data: 13 13 12 8 6 8 9 8 5 12/30 Write block 138 len 24576 data: 6 7 8 8 10 9 13/30 Write block 349 len 16384 data: 8 7 10 12 14/30 Read block 549 len 36864 15/30 Read block 233 len 4096 16/30 Read block 426 len 40960 17/30 Write block 578 len 24576 data: 9 10 9 10 9 8 18/30 Flush 19/30 Write block 300 len 8192 data: 12 10 20/30 Write block 184 len 12288 data: 7 7 6 21/30 Write block 538 len 8192 data: 9 9 22/30 Read block 71 len 8192 23/30 Write block 479 len 36864 data: 9 7 7 6 6 9 7 7 6 24/30 Write block 389 len 32768 data: 5 4 6 5 6 7 7 7 25/30 Read block 185 len 28672 26/30 Write block 396 len 36864 data: 8 8 10 10 11 8 5 5 5 27/30 Read block 31 len 24576 28/30 Write block 531 len 20480 data: 7 9 10 9 9 29/30 Read block 580 len 12288 30/30 Write block 135 len 40960 data: 8 6 5 7 8 9 9 11 10 10 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:06.153796907Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:06.153963487Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1571,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 197 197 197 F F F 1 020-039 106 106 106 195 195 195 F F F 2 040-059 108 103 108 201 189 201 F F F <--- 3 060-079 95 95 95 176 176 176 F F F 4 080-099 107 107 107 198 198 198 F F F 5 100-119 105 105 105 193 193 193 F F F 6 120-139 108 105 108 201 193 201 T F T <--- 7 140-159 108 107 108 201 198 201 T F T <--- 8 160-179 106 106 106 195 195 195 F F F 9 180-199 106 106 106 195 195 195 T F T <--- 10 200-219 104 104 104 191 191 191 F F F 11 220-239 104 104 104 190 190 190 F F F 12 240-259 104 104 104 192 192 192 F F F 13 260-279 102 102 102 188 188 188 F F F 14 280-299 100 100 100 185 185 185 F F F 15 300-319 107 107 107 198 198 198 T F T <--- 16 320-339 101 101 101 186 186 186 F F F 17 340-359 108 106 108 201 197 201 F F F <--- 18 360-379 104 104 104 190 190 190 F F F 19 380-399 98 98 98 182 182 182 T F T <--- 20 400-419 98 98 98 182 182 182 T F T <--- 21 420-439 101 101 101 186 186 186 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 92 92 92 173 173 173 T F T <--- 24 480-499 108 101 108 201 186 201 T T T <--- 25 500-519 108 104 108 201 191 201 F T F <--- 26 520-539 104 104 104 190 190 190 T F T <--- 27 540-559 108 106 108 199 195 199 F T F <--- 28 560-579 108 105 108 201 193 201 F T F <--- 29 580-599 108 92 108 201 173 201 F F F <--- Max gen: 108, Max flush: 201 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 197 197 197 F F F 1 020-039 106 106 106 195 195 195 F F F 2 040-059 108 108 108 201 201 201 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 107 107 107 198 198 198 F F F 5 100-119 105 105 105 193 193 193 F F F 6 120-139 109 109 109 202 202 202 F F F 7 140-159 109 109 109 202 202 202 F F F 8 160-179 106 106 106 195 195 195 F F F 9 180-199 109 109 109 202 202 202 F F F 10 200-219 104 104 104 191 191 191 F F F 11 220-239 104 104 104 190 190 190 F F F 12 240-259 104 104 104 192 192 192 F F F 13 260-279 102 102 102 188 188 188 F F F 14 280-299 100 100 100 185 185 185 F F F 15 300-319 109 109 109 202 202 202 F F F 16 320-339 101 101 101 186 186 186 F F F 17 340-359 108 108 108 201 201 201 F F F 18 360-379 104 104 104 190 190 190 F F F 19 380-399 109 109 109 202 202 202 F F F 20 400-419 109 109 109 202 202 202 F F F 21 420-439 101 101 101 186 186 186 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 109 109 109 202 202 202 F F F 24 480-499 109 109 109 202 202 202 F F F 25 500-519 108 108 108 201 201 201 F F F 26 520-539 109 109 109 202 202 202 F F F 27 540-559 108 108 108 199 199 199 F F F 28 560-579 108 108 108 201 201 201 F F F 29 580-599 108 108 108 201 201 201 F F F Max gen: 109, Max flush: 202 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.411389522Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.419534042Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.419834317Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.419867723Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.419877042Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.4198888Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579} {"msg":"Crucible f1041d97-0ee9-42e4-b562-3a609f50d1bb has session id: e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420180326Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"Upstairs opts: Upstairs UUID: f1041d97-0ee9-42e4-b562-3a609f50d1bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420203123Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"Crucible stats registered with UUID: f1041d97-0ee9-42e4-b562-3a609f50d1bb","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420213942Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:10.420223311Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420348117Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420363445Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420373524Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420521486Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420537224Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579} {"msg":"f1041d97-0ee9-42e4-b562-3a609f50d1bb active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420569501Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.42057892Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"client":"0","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420588618Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"client":"1","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420598267Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"client":"2","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420607866Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420617745Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420627584Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:64579)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420647202Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"io task","client":"2","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:63080)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.420683717Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"io task","client":"0","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:33577)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.42074435Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"io task","client":"1","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421237033Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"client":"0","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421309394Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"client":"1","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421416142Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"client":"2","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421545007Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421659893Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"[0]R flush_numbers[0..12]: [197, 195, 201, 176, 198, 193, 202, 202, 195, 202, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421722686Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"downstairs","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"[0]R generation[0..12]: [106, 106, 108, 95, 107, 105, 109, 109, 106, 109, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421734255Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"downstairs","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421745343Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"downstairs","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"[1]R flush_numbers[0..12]: [197, 195, 201, 176, 198, 193, 202, 202, 195, 202, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421756142Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"downstairs","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"[1]R generation[0..12]: [106, 106, 108, 95, 107, 105, 109, 109, 106, 109, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421766661Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"downstairs","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.42177753Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"downstairs","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"[2]R flush_numbers[0..12]: [197, 195, 201, 176, 198, 193, 202, 202, 195, 202, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421788308Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"downstairs","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"[2]R generation[0..12]: [106, 106, 108, 95, 107, 105, 109, 109, 106, 109, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421798737Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"downstairs","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421809586Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"downstairs","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421818945Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"downstairs","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421828664Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"downstairs","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421837993Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"downstairs","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421847462Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"":"downstairs","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.42185848Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421867139Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"client":"0","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421876998Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"client":"1","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421886937Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"client":"2","":"downstairs","session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421897066Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421906395Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"f1041d97-0ee9-42e4-b562-3a609f50d1bb is now active with session: e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421916353Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421925102Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"session_id":"e5b0d9d5-dcdb-4935-bd16-5b7813fbe8f8"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.421934301Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579} {"msg":"Activated sub_volume f1041d97-0ee9-42e4-b562-3a609f50d1bb","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.42194228Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.4219504Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.422235266Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"task":"crutest"} 01/30 Write block 84 len 24576 data: 11 11 13 11 11 13 02/30 Read block 363 len 4096 03/30 Write block 327 len 4096 data: 7 04/30 Write block 595 len 20480 data: 7 6 4 4 3 05/30 Flush 06/30 Write block 106 len 24576 data: 7 6 6 4 4 4 07/30 Write block 504 len 24576 data: 7 9 10 10 7 6 08/30 Write block 481 len 32768 data: 8 7 7 10 8 8 7 10 09/30 Write block 521 len 20480 data: 7 7 7 6 6 10/30 Flush 11/30 Write block 247 len 40960 data: 9 9 7 9 10 10 9 8 7 9 12/30 Write block 383 len 40960 data: 11 9 7 8 7 6 6 5 7 6 13/30 Read block 282 len 4096 14/30 Read block 56 len 28672 15/30 Read block 311 len 4096 16/30 Write block 303 len 32768 data: 12 11 15 14 14 11 10 10 17/30 Write block 505 len 20480 data: 10 11 11 8 7 18/30 Read block 498 len 12288 19/30 Read block 130 len 28672 20/30 Read block 34 len 4096 21/30 Read block 573 len 4096 22/30 Write block 38 len 36864 data: 6 6 9 8 7 6 8 10 11 23/30 Write block 137 len 16384 data: 6 8 9 10 24/30 Write block 178 len 24576 data: 10 10 10 9 7 8 25/30 Read block 97 len 12288 26/30 Write block 89 len 28672 data: 14 13 11 12 13 13 12 27/30 Read block 479 len 16384 28/30 Write block 326 len 24576 data: 8 8 9 9 10 8 29/30 Write block 296 len 32768 data: 11 12 12 10 13 11 11 13 30/30 Write block 37 len 28672 data: 7 7 7 10 9 8 7 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 1 Done Done Sent false 1003 Acked Write 5 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 6 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Write 5 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Write 7 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Write 6 Done Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 3 28 33 Done 28 27 2 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:10.430224614Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:10.430404183Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1579,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 197 197 197 F F F 1 020-039 106 106 106 195 195 195 T T F <--- 2 040-059 108 108 108 201 201 201 T T F <--- 3 060-079 95 95 95 176 176 176 F F F 4 080-099 110 110 107 203 203 198 T T T <--- 5 100-119 110 110 105 204 204 193 F F F <--- 6 120-139 109 109 109 202 202 202 T T F <--- 7 140-159 109 109 109 202 202 202 T T F <--- 8 160-179 106 106 106 195 195 195 T T F <--- 9 180-199 109 109 109 202 202 202 T T F <--- 10 200-219 104 104 104 191 191 191 F F F 11 220-239 104 104 104 190 190 190 F F F 12 240-259 104 104 104 192 192 192 T T F <--- 13 260-279 102 102 102 188 188 188 F F F 14 280-299 100 100 100 185 185 185 T T F <--- 15 300-319 109 109 109 202 202 202 T T F <--- 16 320-339 110 110 101 203 203 186 T T F <--- 17 340-359 108 108 108 201 201 201 F F F 18 360-379 104 104 104 190 190 190 F F F 19 380-399 109 109 109 202 202 202 T T F <--- 20 400-419 109 109 109 202 202 202 F F F 21 420-439 101 101 101 186 186 186 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 109 109 109 202 202 202 F F F 24 480-499 110 110 109 204 204 202 F F F <--- 25 500-519 110 110 108 204 204 201 T T F <--- 26 520-539 110 110 109 204 204 202 F F F <--- 27 540-559 108 108 108 199 199 199 F F F 28 560-579 108 108 108 201 201 201 F F F 29 580-599 110 110 108 203 203 201 F F F <--- Max gen: 110, Max flush: 204 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 197 197 197 F F F 1 020-039 111 111 111 205 205 205 F F F 2 040-059 111 111 111 205 205 205 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 111 111 111 205 205 205 F F F 5 100-119 110 110 110 204 204 204 F F F 6 120-139 111 111 111 205 205 205 F F F 7 140-159 111 111 111 205 205 205 F F F 8 160-179 111 111 111 205 205 205 F F F 9 180-199 111 111 111 205 205 205 F F F 10 200-219 104 104 104 191 191 191 F F F 11 220-239 104 104 104 190 190 190 F F F 12 240-259 111 111 111 205 205 205 F F F 13 260-279 102 102 102 188 188 188 F F F 14 280-299 111 111 111 205 205 205 F F F 15 300-319 111 111 111 205 205 205 F F F 16 320-339 111 111 111 205 205 205 F F F 17 340-359 108 108 108 201 201 201 F F F 18 360-379 104 104 104 190 190 190 F F F 19 380-399 111 111 111 205 205 205 F F F 20 400-419 109 109 109 202 202 202 F F F 21 420-439 101 101 101 186 186 186 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 109 109 109 202 202 202 F F F 24 480-499 110 110 110 204 204 204 F F F 25 500-519 111 111 111 205 205 205 F F F 26 520-539 110 110 110 204 204 204 F F F 27 540-559 108 108 108 199 199 199 F F F 28 560-579 108 108 108 201 201 201 F F F 29 580-599 110 110 110 203 203 203 F F F Max gen: 111, Max flush: 205 Begin loop 56 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.693392898Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.701458407Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.701729795Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.701753473Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.701765631Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.70177707Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587} {"msg":"Crucible 700531c4-a8ba-43e1-ae40-d7c012e16168 has session id: d9644897-c72c-43d3-a42b-9a6c994a55a2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702071695Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"Upstairs opts: Upstairs UUID: 700531c4-a8ba-43e1-ae40-d7c012e16168, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.70211923Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"Crucible stats registered with UUID: 700531c4-a8ba-43e1-ae40-d7c012e16168","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702131008Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:14.702140367Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702284851Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702313737Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702324576Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702436923Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702451181Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587} {"msg":"700531c4-a8ba-43e1-ae40-d7c012e16168 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702486517Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702496116Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"client":"0","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702505995Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"client":"1","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702517613Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"client":"2","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702527312Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702537291Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702557889Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:48075)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702602763Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"io task","client":"1","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:41893)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.702613312Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"io task","client":"0","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"ds_connection connected from Ok(127.0.0.1:58496)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.70263293Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"io task","client":"2","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703264386Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"client":"2","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703290173Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"client":"0","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703325849Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"client":"1","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703601847Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703625924Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"[0]R flush_numbers[0..12]: [197, 205, 205, 176, 205, 204, 205, 205, 205, 205, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703697466Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"downstairs","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"[0]R generation[0..12]: [106, 111, 111, 95, 111, 110, 111, 111, 111, 111, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703709304Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"downstairs","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703720293Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"downstairs","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"[1]R flush_numbers[0..12]: [197, 205, 205, 176, 205, 204, 205, 205, 205, 205, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703733871Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"downstairs","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"[1]R generation[0..12]: [106, 111, 111, 95, 111, 110, 111, 111, 111, 111, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.70374463Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"downstairs","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703755799Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"downstairs","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"[2]R flush_numbers[0..12]: [197, 205, 205, 176, 205, 204, 205, 205, 205, 205, 191, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703766468Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"downstairs","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"[2]R generation[0..12]: [106, 111, 111, 95, 111, 110, 111, 111, 111, 111, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703776946Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"downstairs","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703787955Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"downstairs","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703797374Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"downstairs","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703807063Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"downstairs","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703816382Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"downstairs","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703825921Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"":"downstairs","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.70383563Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703844418Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"client":"0","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703853977Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"client":"1","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703863706Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"client":"2","":"downstairs","session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703874835Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703884004Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"700531c4-a8ba-43e1-ae40-d7c012e16168 is now active with session: d9644897-c72c-43d3-a42b-9a6c994a55a2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703894013Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703902782Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"session_id":"d9644897-c72c-43d3-a42b-9a6c994a55a2"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703911871Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587} {"msg":"Activated sub_volume 700531c4-a8ba-43e1-ae40-d7c012e16168","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.70391974Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.703927789Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.704202027Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"task":"crutest"} 01/30 Write block 369 len 16384 data: 5 4 5 4 02/30 Write block 24 len 12288 data: 10 9 13 03/30 Write block 194 len 20480 data: 10 10 9 9 10 04/30 Read block 579 len 12288 05/30 Read block 205 len 24576 06/30 Read block 289 len 20480 07/30 Write block 267 len 20480 data: 10 10 11 13 11 08/30 Write block 509 len 16384 data: 8 7 9 9 09/30 Write block 91 len 24576 data: 12 13 14 14 13 10 10/30 Read block 422 len 32768 11/30 Write block 203 len 28672 data: 8 9 8 7 6 8 10 12/30 Write block 481 len 4096 data: 9 13/30 Flush 14/30 Write block 173 len 16384 data: 8 6 5 7 15/30 Read block 198 len 28672 16/30 Read block 304 len 36864 17/30 Write block 269 len 4096 data: 12 18/30 Flush 19/30 Write block 403 len 8192 data: 6 6 20/30 Write block 579 len 4096 data: 11 21/30 Write block 17 len 20480 data: 8 7 10 10 9 22/30 Write block 258 len 16384 data: 5 5 5 5 23/30 Read block 110 len 12288 24/30 Write block 481 len 8192 data: 10 8 25/30 Read block 239 len 8192 26/30 Read block 169 len 4096 27/30 Write block 268 len 12288 data: 11 13 14 28/30 Write block 223 len 24576 data: 9 9 9 9 10 10 29/30 Write block 558 len 12288 data: 9 9 6 30/30 Read block 65 len 32768 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Sent Done false 1001 Acked Write 3 Done Sent Done false 1002 Acked Write 5 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Write 1 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Write 3 Done Sent Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 2 32 Done 30 0 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:14.711671475Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:14.711850194Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1587,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 197 197 197 T F T <--- 1 020-039 112 111 112 206 205 206 T F T <--- 2 040-059 111 111 111 205 205 205 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 112 111 112 206 205 206 F F F <--- 5 100-119 110 110 110 204 204 204 F F F 6 120-139 111 111 111 205 205 205 F F F 7 140-159 111 111 111 205 205 205 F F F 8 160-179 112 111 112 207 205 207 F F F <--- 9 180-199 112 111 112 206 205 206 F F F <--- 10 200-219 112 104 112 206 191 206 F F F <--- 11 220-239 104 104 104 190 190 190 T F T <--- 12 240-259 111 111 111 205 205 205 T F T <--- 13 260-279 112 102 112 207 188 207 T F T <--- 14 280-299 111 111 111 205 205 205 F F F 15 300-319 111 111 111 205 205 205 F F F 16 320-339 111 111 111 205 205 205 F F F 17 340-359 108 108 108 201 201 201 F F F 18 360-379 112 104 112 206 190 206 F F F <--- 19 380-399 111 111 111 205 205 205 F F F 20 400-419 109 109 109 202 202 202 T F T <--- 21 420-439 101 101 101 186 186 186 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 109 109 109 202 202 202 F F F 24 480-499 112 110 112 206 204 206 T F T <--- 25 500-519 112 111 112 206 205 206 F F F <--- 26 520-539 110 110 110 204 204 204 F F F 27 540-559 108 108 108 199 199 199 T F T <--- 28 560-579 108 108 108 201 201 201 T F T <--- 29 580-599 110 110 110 203 203 203 F F F Max gen: 112, Max flush: 207 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 208 208 208 F F F 1 020-039 113 113 113 208 208 208 F F F 2 040-059 111 111 111 205 205 205 F F F 3 060-079 95 95 95 176 176 176 F F F 4 080-099 112 112 112 206 206 206 F F F 5 100-119 110 110 110 204 204 204 F F F 6 120-139 111 111 111 205 205 205 F F F 7 140-159 111 111 111 205 205 205 F F F 8 160-179 112 112 112 207 207 207 F F F 9 180-199 112 112 112 206 206 206 F F F 10 200-219 112 112 112 206 206 206 F F F 11 220-239 113 113 113 208 208 208 F F F 12 240-259 113 113 113 208 208 208 F F F 13 260-279 113 113 113 208 208 208 F F F 14 280-299 111 111 111 205 205 205 F F F 15 300-319 111 111 111 205 205 205 F F F 16 320-339 111 111 111 205 205 205 F F F 17 340-359 108 108 108 201 201 201 F F F 18 360-379 112 112 112 206 206 206 F F F 19 380-399 111 111 111 205 205 205 F F F 20 400-419 113 113 113 208 208 208 F F F 21 420-439 101 101 101 186 186 186 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 109 109 109 202 202 202 F F F 24 480-499 113 113 113 208 208 208 F F F 25 500-519 112 112 112 206 206 206 F F F 26 520-539 110 110 110 204 204 204 F F F 27 540-559 113 113 113 208 208 208 F F F 28 560-579 113 113 113 208 208 208 F F F 29 580-599 110 110 110 203 203 203 F F F Max gen: 113, Max flush: 208 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.965699786Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.973929855Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.974220221Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.974247018Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.974255637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.974267256Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595} {"msg":"Crucible 317d5918-1139-4aee-b564-8364d69f49e4 has session id: a3d3ace7-e7f2-4a61-8c04-578723b61266","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.9745739Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"Upstairs opts: Upstairs UUID: 317d5918-1139-4aee-b564-8364d69f49e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.974603877Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"Crucible stats registered with UUID: 317d5918-1139-4aee-b564-8364d69f49e4","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.974617565Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:18.974627084Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.974769777Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.974785215Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.974795274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.974934388Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.974949396Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595} {"msg":"317d5918-1139-4aee-b564-8364d69f49e4 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.974982232Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.974991691Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"client":"0","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.97500133Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"client":"1","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.975011309Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"client":"2","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.975020938Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.975030937Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.975041086Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"ds_connection connected from Ok(127.0.0.1:59701)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.97508734Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"io task","client":"2","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"ds_connection connected from Ok(127.0.0.1:53912)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.975099979Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"io task","client":"1","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"ds_connection connected from Ok(127.0.0.1:34313)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.975118876Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"io task","client":"0","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.975707998Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"client":"2","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.975825224Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"client":"0","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.975967327Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"client":"1","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976042229Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976122059Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"[0]R flush_numbers[0..12]: [208, 208, 205, 176, 206, 204, 205, 205, 207, 206, 206, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976256574Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"downstairs","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"[0]R generation[0..12]: [113, 113, 111, 95, 112, 110, 111, 111, 112, 112, 112, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976267802Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"downstairs","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976278921Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"downstairs","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"[1]R flush_numbers[0..12]: [208, 208, 205, 176, 206, 204, 205, 205, 207, 206, 206, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.97628959Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"downstairs","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"[1]R generation[0..12]: [113, 113, 111, 95, 112, 110, 111, 111, 112, 112, 112, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976300239Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"downstairs","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976311167Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"downstairs","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"[2]R flush_numbers[0..12]: [208, 208, 205, 176, 206, 204, 205, 205, 207, 206, 206, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976321796Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"downstairs","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"[2]R generation[0..12]: [113, 113, 111, 95, 112, 110, 111, 111, 112, 112, 112, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976334115Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"downstairs","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976345083Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"downstairs","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976354492Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"downstairs","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976364231Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"downstairs","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.97637355Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"downstairs","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976383039Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"":"downstairs","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976392728Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976401547Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"client":"0","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976411076Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"client":"1","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976420685Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"client":"2","":"downstairs","session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976430373Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976439452Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"317d5918-1139-4aee-b564-8364d69f49e4 is now active with session: a3d3ace7-e7f2-4a61-8c04-578723b61266","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976449371Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.97645816Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"session_id":"a3d3ace7-e7f2-4a61-8c04-578723b61266"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976467309Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595} {"msg":"Activated sub_volume 317d5918-1139-4aee-b564-8364d69f49e4","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976475368Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976484637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.976775453Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"task":"crutest"} 01/30 Write block 209 len 8192 data: 11 6 02/30 Flush 03/30 Write block 54 len 20480 data: 11 11 13 9 9 04/30 Write block 344 len 32768 data: 6 5 6 6 7 9 8 11 05/30 Write block 60 len 40960 data: 6 8 7 7 8 8 7 7 7 7 06/30 Read block 70 len 12288 07/30 Read block 497 len 24576 08/30 Write block 168 len 24576 data: 9 8 5 7 8 9 09/30 Write block 554 len 28672 data: 13 13 11 9 10 10 7 10/30 Flush 11/30 Write block 153 len 32768 data: 14 12 12 11 11 10 14 12 12/30 Read block 157 len 36864 13/30 Flush 14/30 Flush 15/30 Write block 301 len 12288 data: 12 12 14 16/30 Read block 501 len 4096 17/30 Flush 18/30 Write block 199 len 20480 data: 7 7 7 8 9 19/30 Flush 20/30 Write block 415 len 40960 data: 7 9 8 8 9 10 10 8 7 5 21/30 Read block 566 len 12288 22/30 Write block 552 len 24576 data: 14 15 14 14 12 10 23/30 Flush 24/30 Flush 25/30 Write block 478 len 8192 data: 9 10 26/30 Write block 495 len 24576 data: 14 13 9 7 9 10 27/30 Write block 346 len 36864 data: 7 7 8 10 9 12 13 13 14 28/30 Write block 401 len 16384 data: 9 6 7 7 29/30 Read block 468 len 12288 30/30 Write block 178 len 24576 data: 11 11 11 10 8 9 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 5 Done Done Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Read 3 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 10 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 27 1 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1001)) Some(JobId(1023)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:18.985182082Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:18.985359861Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1595,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 208 208 208 F F F 1 020-039 113 113 113 208 208 208 F F F 2 040-059 114 111 114 210 205 210 F T F <--- 3 060-079 114 95 114 210 176 210 F F F <--- 4 080-099 112 112 112 206 206 206 F F F 5 100-119 110 110 110 204 204 204 F F F 6 120-139 111 111 111 205 205 205 F F F 7 140-159 114 111 114 211 205 211 F F F <--- 8 160-179 114 112 114 211 207 211 T F T <--- 9 180-199 114 112 114 214 206 214 T F T <--- 10 200-219 114 114 114 214 209 214 F F F <--- 11 220-239 113 113 113 208 208 208 F F F 12 240-259 113 113 113 208 208 208 F F F 13 260-279 113 113 113 208 208 208 F F F 14 280-299 111 111 111 205 205 205 F F F 15 300-319 114 111 114 213 205 213 F F F <--- 16 320-339 111 111 111 205 205 205 F F F 17 340-359 114 108 114 210 201 210 T F T <--- 18 360-379 112 112 112 206 206 206 F F F 19 380-399 111 111 111 205 205 205 F F F 20 400-419 114 113 114 215 208 215 T F T <--- 21 420-439 114 101 114 215 186 215 F F F <--- 22 440-459 96 96 96 177 177 177 F F F 23 460-479 109 109 109 202 202 202 T F T <--- 24 480-499 113 113 113 208 208 208 T F T <--- 25 500-519 112 112 112 206 206 206 T F T <--- 26 520-539 110 110 110 204 204 204 F F F 27 540-559 114 113 114 215 208 215 F F F <--- 28 560-579 114 113 114 210 208 210 F F F <--- 29 580-599 110 110 110 203 203 203 F F F Max gen: 114, Max flush: 215 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 208 208 208 F F F 1 020-039 113 113 113 208 208 208 F F F 2 040-059 114 114 114 210 210 210 F F F 3 060-079 114 114 114 210 210 210 F F F 4 080-099 112 112 112 206 206 206 F F F 5 100-119 110 110 110 204 204 204 F F F 6 120-139 111 111 111 205 205 205 F F F 7 140-159 114 114 114 211 211 211 F F F 8 160-179 115 115 115 216 216 216 F F F 9 180-199 115 115 115 216 216 216 F F F 10 200-219 114 114 114 214 214 214 F F F 11 220-239 113 113 113 208 208 208 F F F 12 240-259 113 113 113 208 208 208 F F F 13 260-279 113 113 113 208 208 208 F F F 14 280-299 111 111 111 205 205 205 F F F 15 300-319 114 114 114 213 213 213 F F F 16 320-339 111 111 111 205 205 205 F F F 17 340-359 115 115 115 216 216 216 F F F 18 360-379 112 112 112 206 206 206 F F F 19 380-399 111 111 111 205 205 205 F F F 20 400-419 115 115 115 216 216 216 F F F 21 420-439 114 114 114 215 215 215 F F F 22 440-459 96 96 96 177 177 177 F F F 23 460-479 115 115 115 216 216 216 F F F 24 480-499 115 115 115 216 216 216 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 110 110 110 204 204 204 F F F 27 540-559 114 114 114 215 215 215 F F F 28 560-579 114 114 114 210 210 210 F F F 29 580-599 110 110 110 203 203 203 F F F Max gen: 115, Max flush: 216 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.238627382Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.246930042Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.247214859Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.247241106Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.247250115Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.247261704Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603} {"msg":"Crucible c5876586-56b0-4b7a-b1a9-5b520a0c69a4 has session id: fe6574d1-d66a-48db-a526-c9400c3d9b8c","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.247627701Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"Upstairs opts: Upstairs UUID: c5876586-56b0-4b7a-b1a9-5b520a0c69a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.247657088Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"Crucible stats registered with UUID: c5876586-56b0-4b7a-b1a9-5b520a0c69a4","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.247668396Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:23.247677755Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.247800951Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.247816319Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.247828787Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248001797Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248018805Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603} {"msg":"c5876586-56b0-4b7a-b1a9-5b520a0c69a4 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248041953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248051401Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"client":"0","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.24806121Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"client":"1","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248071109Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"client":"2","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248080838Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248091017Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248143891Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42722)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248169468Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"io task","client":"1","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55604)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248180796Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"io task","client":"2","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55763)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248206773Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"io task","client":"0","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248882454Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"client":"0","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248927129Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"client":"1","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.248949317Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"client":"2","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249213616Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249236063Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"[0]R flush_numbers[0..12]: [208, 208, 210, 210, 206, 204, 205, 211, 216, 216, 214, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249246792Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"downstairs","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"[0]R generation[0..12]: [113, 113, 114, 114, 112, 110, 111, 114, 115, 115, 114, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249257841Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"downstairs","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249268819Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"downstairs","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"[1]R flush_numbers[0..12]: [208, 208, 210, 210, 206, 204, 205, 211, 216, 216, 214, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249279478Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"downstairs","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"[1]R generation[0..12]: [113, 113, 114, 114, 112, 110, 111, 114, 115, 115, 114, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249290007Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"downstairs","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249300966Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"downstairs","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"[2]R flush_numbers[0..12]: [208, 208, 210, 210, 206, 204, 205, 211, 216, 216, 214, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249311394Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"downstairs","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"[2]R generation[0..12]: [113, 113, 114, 114, 112, 110, 111, 114, 115, 115, 114, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249321973Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"downstairs","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249332932Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"downstairs","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249342311Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"downstairs","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.24935199Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"downstairs","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249361689Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"downstairs","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249373767Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"":"downstairs","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249383846Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249392745Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"client":"0","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249402584Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"client":"1","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249412293Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"client":"2","":"downstairs","session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249422061Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.24943106Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"c5876586-56b0-4b7a-b1a9-5b520a0c69a4 is now active with session: fe6574d1-d66a-48db-a526-c9400c3d9b8c","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249441049Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249449828Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"session_id":"fe6574d1-d66a-48db-a526-c9400c3d9b8c"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249458987Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603} {"msg":"Activated sub_volume c5876586-56b0-4b7a-b1a9-5b520a0c69a4","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249467076Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249475185Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.249830594Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"task":"crutest"} 01/30 Write block 389 len 32768 data: 7 6 8 7 7 8 8 9 02/30 Read block 588 len 4096 03/30 Write block 487 len 32768 data: 8 11 9 11 11 15 14 14 04/30 Write block 431 len 4096 data: 10 05/30 Write block 252 len 16384 data: 11 10 9 8 06/30 Write block 548 len 4096 data: 16 07/30 Write block 448 len 28672 data: 7 6 8 5 3 3 5 08/30 Read block 150 len 24576 09/30 Read block 243 len 4096 10/30 Read block 523 len 36864 11/30 Read block 592 len 16384 12/30 Write block 183 len 40960 data: 10 8 8 7 4 5 10 11 13 10 13/30 Read block 29 len 12288 14/30 Read block 93 len 28672 15/30 Write block 23 len 8192 data: 9 11 16/30 Write block 317 len 32768 data: 12 13 15 13 14 11 8 9 17/30 Write block 404 len 32768 data: 8 5 4 5 5 6 6 5 18/30 Flush 19/30 Write block 330 len 20480 data: 11 9 11 12 11 20/30 Write block 298 len 16384 data: 13 11 14 13 21/30 Write block 22 len 20480 data: 11 10 12 10 14 22/30 Write block 312 len 16384 data: 5 5 5 8 23/30 Write block 386 len 40960 data: 9 8 7 8 7 9 8 8 9 9 24/30 Read block 142 len 40960 25/30 Read block 344 len 28672 26/30 Read block 495 len 36864 27/30 Write block 143 len 12288 data: 11 11 11 28/30 Write block 364 len 12288 data: 7 6 5 29/30 Read block 124 len 16384 30/30 Write block 97 len 40960 data: 6 7 6 7 6 6 5 6 4 8 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 8 Done Done Sent false 1003 Acked Write 1 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Write 4 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Read 10 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:23.257710104Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:23.257884764Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1603,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 208 208 208 F F F 1 020-039 116 116 113 217 217 208 T T F <--- 2 040-059 114 114 114 210 210 210 F F F 3 060-079 114 114 114 210 210 210 F F F 4 080-099 112 112 112 206 206 206 T T F <--- 5 100-119 110 110 110 204 204 204 T T F <--- 6 120-139 111 111 111 205 205 205 F F F 7 140-159 114 114 114 211 211 211 T T F <--- 8 160-179 115 115 115 216 216 216 F F F 9 180-199 116 116 115 217 217 216 F F F <--- 10 200-219 114 114 114 214 214 214 F F F 11 220-239 113 113 113 208 208 208 F F F 12 240-259 116 116 113 217 217 208 F F F <--- 13 260-279 113 113 113 208 208 208 F F F 14 280-299 111 111 111 205 205 205 T T F <--- 15 300-319 116 116 114 217 217 213 T T F <--- 16 320-339 116 116 111 217 217 205 T T F <--- 17 340-359 115 115 115 216 216 216 F F F 18 360-379 112 112 112 206 206 206 T T F <--- 19 380-399 116 116 111 217 217 205 T T T <--- 20 400-419 116 116 115 217 217 216 F F F <--- 21 420-439 116 116 114 217 217 215 F F F <--- 22 440-459 116 116 96 217 217 177 F F F <--- 23 460-479 115 115 115 216 216 216 F F F 24 480-499 116 116 115 217 217 216 F F F <--- 25 500-519 115 115 115 216 216 216 F F F 26 520-539 110 110 110 204 204 204 F F F 27 540-559 116 116 114 217 217 215 F F F <--- 28 560-579 114 114 114 210 210 210 F F F 29 580-599 110 110 110 203 203 203 F F F Max gen: 116, Max flush: 217 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 113 113 208 208 208 F F F 1 020-039 117 117 117 218 218 218 F F F 2 040-059 114 114 114 210 210 210 F F F 3 060-079 114 114 114 210 210 210 F F F 4 080-099 117 117 117 218 218 218 F F F 5 100-119 117 117 117 218 218 218 F F F 6 120-139 111 111 111 205 205 205 F F F 7 140-159 117 117 117 218 218 218 F F F 8 160-179 115 115 115 216 216 216 F F F 9 180-199 116 116 116 217 217 217 F F F 10 200-219 114 114 114 214 214 214 F F F 11 220-239 113 113 113 208 208 208 F F F 12 240-259 116 116 116 217 217 217 F F F 13 260-279 113 113 113 208 208 208 F F F 14 280-299 117 117 117 218 218 218 F F F 15 300-319 117 117 117 218 218 218 F F F 16 320-339 117 117 117 218 218 218 F F F 17 340-359 115 115 115 216 216 216 F F F 18 360-379 117 117 117 218 218 218 F F F 19 380-399 117 117 117 218 218 218 F F F 20 400-419 116 116 116 217 217 217 F F F 21 420-439 116 116 116 217 217 217 F F F 22 440-459 116 116 116 217 217 217 F F F 23 460-479 115 115 115 216 216 216 F F F 24 480-499 116 116 116 217 217 217 F F F 25 500-519 115 115 115 216 216 216 F F F 26 520-539 110 110 110 204 204 204 F F F 27 540-559 116 116 116 217 217 217 F F F 28 560-579 114 114 114 210 210 210 F F F 29 580-599 110 110 110 203 203 203 F F F Max gen: 117, Max flush: 218 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.517030388Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:27.524286681Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.524340915Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.531372944Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.531617405Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.531641662Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.531650181Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.53166174Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611} {"msg":"Crucible d2447a64-8110-408f-8ccf-531f7ce074f6 has session id: 56f08b25-3af5-4bf2-990f-10803ab14e9b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.531997911Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"Upstairs opts: Upstairs UUID: d2447a64-8110-408f-8ccf-531f7ce074f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532022848Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"Crucible stats registered with UUID: d2447a64-8110-408f-8ccf-531f7ce074f6","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532032557Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:27.532041556Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.53217643Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532192288Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532202437Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532385716Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532401324Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611} {"msg":"d2447a64-8110-408f-8ccf-531f7ce074f6 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532417762Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532428141Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"client":"0","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532437829Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"client":"1","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532447498Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"client":"2","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532456977Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532466896Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532476745Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:38468)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532547527Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"io task","client":"1","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61284)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532673142Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"io task","client":"2","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:65379)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.532697659Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"io task","client":"0","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533153456Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"client":"1","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533172234Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"client":"2","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533220638Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"client":"0","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533487947Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533519183Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"[0]R flush_numbers[0..12]: [208, 218, 210, 210, 218, 218, 205, 218, 216, 217, 214, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533563628Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"downstairs","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"[0]R generation[0..12]: [113, 117, 114, 114, 117, 117, 111, 117, 115, 116, 114, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533578166Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"downstairs","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533589405Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"downstairs","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"[1]R flush_numbers[0..12]: [208, 218, 210, 210, 218, 218, 205, 218, 216, 217, 214, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533600224Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"downstairs","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"[1]R generation[0..12]: [113, 117, 114, 114, 117, 117, 111, 117, 115, 116, 114, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533610913Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"downstairs","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533621881Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"downstairs","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"[2]R flush_numbers[0..12]: [208, 218, 210, 210, 218, 218, 205, 218, 216, 217, 214, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.53363265Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"downstairs","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"[2]R generation[0..12]: [113, 117, 114, 114, 117, 117, 111, 117, 115, 116, 114, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533643479Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"downstairs","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533654537Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"downstairs","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533664076Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"downstairs","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533673935Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"downstairs","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"Next flush: 219","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533683254Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"downstairs","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533692803Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"":"downstairs","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533702902Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533711781Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"client":"0","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533723029Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"client":"1","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533732978Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"client":"2","":"downstairs","session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533743117Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533752396Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"d2447a64-8110-408f-8ccf-531f7ce074f6 is now active with session: 56f08b25-3af5-4bf2-990f-10803ab14e9b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533767324Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533776153Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"session_id":"56f08b25-3af5-4bf2-990f-10803ab14e9b"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533785282Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611} {"msg":"Activated sub_volume d2447a64-8110-408f-8ccf-531f7ce074f6","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.533793301Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.53380132Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.53406461Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"task":"crutest"} 01/30 Write block 248 len 4096 data: 10 02/30 Write block 443 len 12288 data: 9 10 11 03/30 Read block 305 len 32768 04/30 Read block 491 len 32768 05/30 Write block 519 len 36864 data: 11 8 8 8 8 7 7 4 4 06/30 Write block 247 len 20480 data: 10 11 8 10 11 07/30 Flush 08/30 Write block 242 len 4096 data: 15 09/30 Read block 227 len 24576 10/30 Read block 411 len 24576 11/30 Write block 2 len 28672 data: 6 7 11 11 12 11 12 12/30 Read block 587 len 36864 13/30 Write block 50 len 12288 data: 8 13 11 14/30 Read block 330 len 28672 15/30 Write block 188 len 12288 data: 6 11 12 16/30 Read block 243 len 32768 17/30 Flush 18/30 Write block 529 len 4096 data: 5 19/30 Read block 318 len 4096 20/30 Read block 518 len 4096 21/30 Read block 591 len 8192 22/30 Flush 23/30 Write block 407 len 36864 data: 6 6 7 7 6 8 7 7 8 24/30 Read block 140 len 40960 25/30 Read block 450 len 24576 26/30 Write block 406 len 12288 data: 5 7 7 27/30 Write block 597 len 4096 data: 5 28/30 Write block 409 len 12288 data: 8 8 7 29/30 Write block 241 len 12288 data: 15 16 13 30/30 Read block 51 len 20480 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Sent Done Done false 1001 Acked Write 3 Sent Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Read 8 Sent Done Done false 1004 Acked Write 9 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:27.542183542Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:27.542362471Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1611,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 113 118 118 208 220 220 F F F <--- 1 020-039 117 117 117 218 218 218 F F F 2 040-059 114 118 118 210 220 220 F F F <--- 3 060-079 114 114 114 210 210 210 F F F 4 080-099 117 117 117 218 218 218 F F F 5 100-119 117 117 117 218 218 218 F F F 6 120-139 111 111 111 205 205 205 F F F 7 140-159 117 117 117 218 218 218 F F F 8 160-179 115 115 115 216 216 216 F F F 9 180-199 116 118 118 217 220 220 F F F <--- 10 200-219 114 114 114 214 214 214 F F F 11 220-239 113 113 113 208 208 208 F F F 12 240-259 116 118 118 217 220 220 F T T <--- 13 260-279 113 113 113 208 208 208 F F F 14 280-299 117 117 117 218 218 218 F F F 15 300-319 117 117 117 218 218 218 F F F 16 320-339 117 117 117 218 218 218 F F F 17 340-359 115 115 115 216 216 216 F F F 18 360-379 117 117 117 218 218 218 F F F 19 380-399 117 117 117 218 218 218 F F F 20 400-419 116 116 116 217 217 217 F T T <--- 21 420-439 116 116 116 217 217 217 F F F 22 440-459 116 118 118 217 219 219 F F F <--- 23 460-479 115 115 115 216 216 216 F F F 24 480-499 116 116 116 217 217 217 F F F 25 500-519 115 118 118 216 219 219 F F F <--- 26 520-539 110 118 118 204 221 221 F F F <--- 27 540-559 116 116 116 217 217 217 F F F 28 560-579 114 114 114 210 210 210 F F F 29 580-599 110 110 110 203 203 203 F T T <--- Max gen: 118, Max flush: 221 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 220 220 220 F F F 1 020-039 117 117 117 218 218 218 F F F 2 040-059 118 118 118 220 220 220 F F F 3 060-079 114 114 114 210 210 210 F F F 4 080-099 117 117 117 218 218 218 F F F 5 100-119 117 117 117 218 218 218 F F F 6 120-139 111 111 111 205 205 205 F F F 7 140-159 117 117 117 218 218 218 F F F 8 160-179 115 115 115 216 216 216 F F F 9 180-199 118 118 118 220 220 220 F F F 10 200-219 114 114 114 214 214 214 F F F 11 220-239 113 113 113 208 208 208 F F F 12 240-259 119 119 119 222 222 222 F F F 13 260-279 113 113 113 208 208 208 F F F 14 280-299 117 117 117 218 218 218 F F F 15 300-319 117 117 117 218 218 218 F F F 16 320-339 117 117 117 218 218 218 F F F 17 340-359 115 115 115 216 216 216 F F F 18 360-379 117 117 117 218 218 218 F F F 19 380-399 117 117 117 218 218 218 F F F 20 400-419 119 119 119 222 222 222 F F F 21 420-439 116 116 116 217 217 217 F F F 22 440-459 118 118 118 219 219 219 F F F 23 460-479 115 115 115 216 216 216 F F F 24 480-499 116 116 116 217 217 217 F F F 25 500-519 118 118 118 219 219 219 F F F 26 520-539 118 118 118 221 221 221 F F F 27 540-559 116 116 116 217 217 217 F F F 28 560-579 114 114 114 210 210 210 F F F 29 580-599 119 119 119 222 222 222 F F F Max gen: 119, Max flush: 222 Begin loop 60 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.79058484Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.798755917Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799041303Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.79906954Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799078149Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799089618Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619} {"msg":"Crucible 2e9a5bb9-90e9-4cb5-adfd-a0df12625e99 has session id: 48114790-2958-419b-a616-857b6f66172b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799362046Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"Upstairs opts: Upstairs UUID: 2e9a5bb9-90e9-4cb5-adfd-a0df12625e99, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799471033Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"Crucible stats registered with UUID: 2e9a5bb9-90e9-4cb5-adfd-a0df12625e99","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.79949583Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:31.799506489Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799609717Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799624925Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799635134Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799805464Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799824442Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619} {"msg":"2e9a5bb9-90e9-4cb5-adfd-a0df12625e99 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799847609Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799856918Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"client":"0","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799866577Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"client":"1","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799876216Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"client":"2","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799885615Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799895484Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799909142Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46099)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799936629Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"io task","client":"2","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45500)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799950317Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"io task","client":"0","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"ds_connection connected from Ok(127.0.0.1:34090)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.799968105Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"io task","client":"1","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.800596352Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"client":"2","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.800617409Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"client":"0","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.800635757Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"client":"1","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.800925623Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.800969718Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"[0]R flush_numbers[0..12]: [220, 218, 220, 210, 218, 218, 205, 218, 216, 220, 214, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801020752Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"downstairs","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"[0]R generation[0..12]: [118, 117, 118, 114, 117, 117, 111, 117, 115, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801032211Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"downstairs","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.80104341Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"downstairs","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"[1]R flush_numbers[0..12]: [220, 218, 220, 210, 218, 218, 205, 218, 216, 220, 214, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801054108Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"downstairs","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"[1]R generation[0..12]: [118, 117, 118, 114, 117, 117, 111, 117, 115, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801064657Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"downstairs","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801075516Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"downstairs","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"[2]R flush_numbers[0..12]: [220, 218, 220, 210, 218, 218, 205, 218, 216, 220, 214, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801088834Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"downstairs","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"[2]R generation[0..12]: [118, 117, 118, 114, 117, 117, 111, 117, 115, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801099343Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"downstairs","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801110262Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"downstairs","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801119781Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"downstairs","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.8011296Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"downstairs","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801139468Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"downstairs","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801148917Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"":"downstairs","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801159126Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801167985Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"client":"0","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801178024Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"client":"1","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801187803Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"client":"2","":"downstairs","session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801197932Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801207Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"2e9a5bb9-90e9-4cb5-adfd-a0df12625e99 is now active with session: 48114790-2958-419b-a616-857b6f66172b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801217009Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801225728Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"session_id":"48114790-2958-419b-a616-857b6f66172b"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801240757Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619} {"msg":"Activated sub_volume 2e9a5bb9-90e9-4cb5-adfd-a0df12625e99","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801248676Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801256815Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.801540082Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"task":"crutest"} 01/30 Flush 02/30 Write block 336 len 24576 data: 11 12 10 11 10 7 03/30 Write block 145 len 16384 data: 12 11 13 14 04/30 Write block 494 len 24576 data: 15 15 14 10 8 10 05/30 Write block 351 len 12288 data: 13 14 14 06/30 Read block 158 len 4096 07/30 Read block 127 len 4096 08/30 Write block 591 len 36864 data: 7 8 8 6 8 7 6 5 4 09/30 Read block 583 len 8192 10/30 Write block 475 len 4096 data: 8 11/30 Read block 341 len 8192 12/30 Write block 555 len 8192 data: 15 13 13/30 Write block 25 len 8192 data: 11 15 14/30 Write block 597 len 8192 data: 7 6 15/30 Read block 133 len 12288 16/30 Read block 499 len 36864 17/30 Read block 394 len 16384 18/30 Read block 208 len 28672 19/30 Write block 313 len 12288 data: 6 6 9 20/30 Flush 21/30 Write block 53 len 32768 data: 11 12 12 14 10 10 7 7 22/30 Read block 443 len 12288 23/30 Flush 24/30 Write block 106 len 28672 data: 9 7 7 5 5 5 5 25/30 Read block 43 len 16384 26/30 Read block 29 len 36864 27/30 Read block 152 len 24576 28/30 Read block 16 len 28672 29/30 Read block 158 len 20480 30/30 Read block 153 len 40960 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 6 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 10 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 0 26 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1000)) Some(JobId(1022)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:31.810091633Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:31.810263023Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1619,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 220 220 220 F F F 1 020-039 120 117 120 224 218 224 F F F <--- 2 040-059 120 118 120 225 220 225 F F F <--- 3 060-079 120 114 120 225 210 225 F F F <--- 4 080-099 117 117 117 218 218 218 F F F 5 100-119 117 117 117 218 218 218 T F T <--- 6 120-139 111 111 111 205 205 205 F F F 7 140-159 120 117 120 224 218 224 F T F <--- 8 160-179 115 115 115 216 216 216 F F F 9 180-199 118 118 118 220 220 220 F F F 10 200-219 114 114 114 214 214 214 F F F 11 220-239 113 113 113 208 208 208 F F F 12 240-259 119 119 119 222 222 222 F F F 13 260-279 113 113 113 208 208 208 F F F 14 280-299 117 117 117 218 218 218 F F F 15 300-319 120 117 120 224 218 224 F F F <--- 16 320-339 120 117 120 224 218 224 F T F <--- 17 340-359 120 115 120 224 216 224 F T F <--- 18 360-379 117 117 117 218 218 218 F F F 19 380-399 117 117 117 218 218 218 F F F 20 400-419 119 119 119 222 222 222 F F F 21 420-439 116 116 116 217 217 217 F F F 22 440-459 118 118 118 219 219 219 F F F 23 460-479 120 115 120 224 216 224 F F F <--- 24 480-499 120 116 120 224 217 224 F T F <--- 25 500-519 118 118 118 219 219 219 F F F 26 520-539 118 118 118 221 221 221 F F F 27 540-559 120 116 120 224 217 224 F F F <--- 28 560-579 114 114 114 210 210 210 F F F 29 580-599 120 119 120 224 222 224 F F F <--- Max gen: 120, Max flush: 225 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 220 220 220 F F F 1 020-039 120 120 120 224 224 224 F F F 2 040-059 120 120 120 225 225 225 F F F 3 060-079 120 120 120 225 225 225 F F F 4 080-099 117 117 117 218 218 218 F F F 5 100-119 121 121 121 226 226 226 F F F 6 120-139 111 111 111 205 205 205 F F F 7 140-159 120 120 120 224 224 224 F F F 8 160-179 115 115 115 216 216 216 F F F 9 180-199 118 118 118 220 220 220 F F F 10 200-219 114 114 114 214 214 214 F F F 11 220-239 113 113 113 208 208 208 F F F 12 240-259 119 119 119 222 222 222 F F F 13 260-279 113 113 113 208 208 208 F F F 14 280-299 117 117 117 218 218 218 F F F 15 300-319 120 120 120 224 224 224 F F F 16 320-339 120 120 120 224 224 224 F F F 17 340-359 120 120 120 224 224 224 F F F 18 360-379 117 117 117 218 218 218 F F F 19 380-399 117 117 117 218 218 218 F F F 20 400-419 119 119 119 222 222 222 F F F 21 420-439 116 116 116 217 217 217 F F F 22 440-459 118 118 118 219 219 219 F F F 23 460-479 120 120 120 224 224 224 F F F 24 480-499 120 120 120 224 224 224 F F F 25 500-519 118 118 118 219 219 219 F F F 26 520-539 118 118 118 221 221 221 F F F 27 540-559 120 120 120 224 224 224 F F F 28 560-579 114 114 114 210 210 210 F F F 29 580-599 120 120 120 224 224 224 F F F Max gen: 121, Max flush: 226 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.058518428Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:36.065587203Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.065643327Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.072568088Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.072834237Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.072860704Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.072869243Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.072880742Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627} {"msg":"Crucible 4fbed61a-57c9-46d7-9ccc-a00a9a7a8eb8 has session id: a66286d6-1768-403a-b439-e183175e1fcf","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073231211Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"Upstairs opts: Upstairs UUID: 4fbed61a-57c9-46d7-9ccc-a00a9a7a8eb8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073252319Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"Crucible stats registered with UUID: 4fbed61a-57c9-46d7-9ccc-a00a9a7a8eb8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073261997Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:36.073273186Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.07341331Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073428758Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073438897Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073595498Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073611007Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627} {"msg":"4fbed61a-57c9-46d7-9ccc-a00a9a7a8eb8 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073642903Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073652312Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"client":"0","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073661921Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"client":"1","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.07367143Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"client":"2","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073681068Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073691237Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073701196Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"ds_connection connected from Ok(127.0.0.1:54486)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073745111Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"io task","client":"2","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"ds_connection connected from Ok(127.0.0.1:33086)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.07375615Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"io task","client":"1","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"ds_connection connected from Ok(127.0.0.1:46766)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.073770298Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"io task","client":"0","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.07444155Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"client":"1","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.074466327Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"client":"2","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.074588123Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"client":"0","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.074758903Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.074768542Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"[0]R flush_numbers[0..12]: [220, 224, 225, 225, 218, 226, 205, 224, 216, 220, 214, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.074902826Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"downstairs","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"[0]R generation[0..12]: [118, 120, 120, 120, 117, 121, 111, 120, 115, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.074914275Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"downstairs","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.074925303Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"downstairs","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"[1]R flush_numbers[0..12]: [220, 224, 225, 225, 218, 226, 205, 224, 216, 220, 214, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.074935912Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"downstairs","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"[1]R generation[0..12]: [118, 120, 120, 120, 117, 121, 111, 120, 115, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.074946591Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"downstairs","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.07495743Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"downstairs","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"[2]R flush_numbers[0..12]: [220, 224, 225, 225, 218, 226, 205, 224, 216, 220, 214, 208]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.074968138Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"downstairs","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"[2]R generation[0..12]: [118, 120, 120, 120, 117, 121, 111, 120, 115, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.074978687Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"downstairs","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.074989876Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"downstairs","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075001574Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"downstairs","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075011393Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"downstairs","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075020952Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"downstairs","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075030501Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"":"downstairs","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.07504067Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075049499Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"client":"0","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075059168Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"client":"1","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075068807Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"client":"2","":"downstairs","session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075078775Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075087974Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"4fbed61a-57c9-46d7-9ccc-a00a9a7a8eb8 is now active with session: a66286d6-1768-403a-b439-e183175e1fcf","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075097903Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075106562Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"session_id":"a66286d6-1768-403a-b439-e183175e1fcf"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075115701Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627} {"msg":"Activated sub_volume 4fbed61a-57c9-46d7-9ccc-a00a9a7a8eb8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.07512368Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075131989Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.075426435Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"task":"crutest"} 01/30 Write block 105 len 32768 data: 5 10 8 8 6 6 6 6 02/30 Flush 03/30 Write block 208 len 24576 data: 9 12 7 5 5 5 04/30 Read block 42 len 20480 05/30 Read block 177 len 32768 06/30 Read block 425 len 8192 07/30 Write block 474 len 24576 data: 9 9 10 10 10 11 08/30 Write block 209 len 20480 data: 13 8 6 6 6 09/30 Read block 164 len 16384 10/30 Read block 499 len 16384 11/30 Write block 528 len 16384 data: 4 6 8 8 12/30 Read block 66 len 12288 13/30 Read block 498 len 24576 14/30 Read block 175 len 12288 15/30 Write block 582 len 4096 data: 10 16/30 Write block 354 len 16384 data: 15 11 8 6 17/30 Write block 579 len 12288 data: 12 10 11 18/30 Read block 591 len 28672 19/30 Write block 77 len 12288 data: 6 9 8 20/30 Read block 160 len 32768 21/30 Write block 223 len 20480 data: 10 10 10 10 11 22/30 Read block 140 len 20480 23/30 Write block 528 len 36864 data: 5 7 9 9 10 11 10 10 10 24/30 Read block 244 len 8192 25/30 Read block 405 len 32768 26/30 Write block 565 len 16384 data: 9 9 10 11 27/30 Read block 492 len 32768 28/30 Write block 328 len 28672 data: 10 10 12 10 12 13 12 29/30 Read block 143 len 40960 30/30 Write block 306 len 32768 data: 15 15 12 11 11 5 6 7 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 6 Done Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Read 2 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Read 5 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Read 10 Sent Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 1 29 Done 1 27 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:36.084417115Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:36.084588425Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1627,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 220 220 220 F F F 1 020-039 120 120 120 224 224 224 F F F 2 040-059 120 120 120 225 225 225 F F F 3 060-079 120 120 120 225 225 225 F T T <--- 4 080-099 117 117 117 218 218 218 F F F 5 100-119 122 122 122 227 227 227 F F F 6 120-139 111 111 111 205 205 205 F F F 7 140-159 120 120 120 224 224 224 F F F 8 160-179 115 115 115 216 216 216 F F F 9 180-199 118 118 118 220 220 220 F F F 10 200-219 114 114 114 214 214 214 T T T 11 220-239 113 113 113 208 208 208 F T T <--- 12 240-259 119 119 119 222 222 222 F F F 13 260-279 113 113 113 208 208 208 F F F 14 280-299 117 117 117 218 218 218 F F F 15 300-319 120 120 120 224 224 224 F T T <--- 16 320-339 120 120 120 224 224 224 F T T <--- 17 340-359 120 120 120 224 224 224 F T T <--- 18 360-379 117 117 117 218 218 218 F F F 19 380-399 117 117 117 218 218 218 F F F 20 400-419 119 119 119 222 222 222 F F F 21 420-439 116 116 116 217 217 217 F F F 22 440-459 118 118 118 219 219 219 F F F 23 460-479 120 120 120 224 224 224 F T T <--- 24 480-499 120 120 120 224 224 224 F F F 25 500-519 118 118 118 219 219 219 F F F 26 520-539 118 118 118 221 221 221 F T T <--- 27 540-559 120 120 120 224 224 224 F F F 28 560-579 114 114 114 210 210 210 F T T <--- 29 580-599 120 120 120 224 224 224 F T T <--- Max gen: 122, Max flush: 227 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 61 Verifying data now Files /tmp/test_repair/test_repair_verify.data and /tmp/test_repair/test_repair_verify.data.last differ 212,216c212,216 < 12, < 7, < 5, < 5, < 5, --- > 13, > 8, > 6, > 6, > 6, diff found after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 220 220 220 F F F 1 020-039 120 120 120 224 224 224 F F F 2 040-059 120 120 120 225 225 225 F F F 3 060-079 123 123 123 228 228 228 F F F 4 080-099 117 117 117 218 218 218 F F F 5 100-119 122 122 122 227 227 227 F F F 6 120-139 111 111 111 205 205 205 F F F 7 140-159 120 120 120 224 224 224 F F F 8 160-179 115 115 115 216 216 216 F F F 9 180-199 118 118 118 220 220 220 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 123 123 123 228 228 228 F F F 12 240-259 119 119 119 222 222 222 F F F 13 260-279 113 113 113 208 208 208 F F F 14 280-299 117 117 117 218 218 218 F F F 15 300-319 123 123 123 228 228 228 F F F 16 320-339 123 123 123 228 228 228 F F F 17 340-359 123 123 123 228 228 228 F F F 18 360-379 117 117 117 218 218 218 F F F 19 380-399 117 117 117 218 218 218 F F F 20 400-419 119 119 119 222 222 222 F F F 21 420-439 116 116 116 217 217 217 F F F 22 440-459 118 118 118 219 219 219 F F F 23 460-479 123 123 123 228 228 228 F F F 24 480-499 120 120 120 224 224 224 F F F 25 500-519 118 118 118 219 219 219 F F F 26 520-539 123 123 123 228 228 228 F F F 27 540-559 120 120 120 224 224 224 F F F 28 560-579 123 123 123 228 228 228 F F F 29 580-599 123 123 123 228 228 228 F F F Max gen: 123, Max flush: 228 Begin loop 62 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.344538916Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.352592986Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.352867923Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.3528935Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.352902409Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.352914228Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638} {"msg":"Crucible 4710e400-63d2-4dd5-8296-e4c293c9c213 has session id: 48e2b089-5d27-4276-993d-39aa0182741a","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353255928Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"Upstairs opts: Upstairs UUID: 4710e400-63d2-4dd5-8296-e4c293c9c213, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353284725Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"Crucible stats registered with UUID: 4710e400-63d2-4dd5-8296-e4c293c9c213","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353295763Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:40.353305042Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353449356Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353478152Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353489001Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353611167Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353626505Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638} {"msg":"4710e400-63d2-4dd5-8296-e4c293c9c213 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353662921Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.35367244Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"client":"0","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353682378Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"client":"1","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353691967Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"client":"2","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353701556Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353711755Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353721674Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"ds_connection connected from Ok(127.0.0.1:34598)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353732043Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"ds_connection connected from Ok(127.0.0.1:65387)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.353784686Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"ds_connection connected from Ok(127.0.0.1:39562)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.35383786Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354378617Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"client":"1","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354409214Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"client":"0","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354502813Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"client":"2","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354702529Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354720877Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"[0]R flush_numbers[0..12]: [220, 224, 225, 228, 218, 227, 205, 224, 216, 220, 228, 228]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354774691Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"downstairs","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"[0]R generation[0..12]: [118, 120, 120, 123, 117, 122, 111, 120, 115, 118, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.35478623Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"downstairs","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354797498Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"downstairs","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"[1]R flush_numbers[0..12]: [220, 224, 225, 228, 218, 227, 205, 224, 216, 220, 228, 228]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354808047Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"downstairs","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"[1]R generation[0..12]: [118, 120, 120, 123, 117, 122, 111, 120, 115, 118, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354818616Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"downstairs","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354829434Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"downstairs","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"[2]R flush_numbers[0..12]: [220, 224, 225, 228, 218, 227, 205, 224, 216, 220, 228, 228]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354839983Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"downstairs","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"[2]R generation[0..12]: [118, 120, 120, 123, 117, 122, 111, 120, 115, 118, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354850392Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"downstairs","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354861361Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"downstairs","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.35487092Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"downstairs","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354880638Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"downstairs","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354890017Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"downstairs","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354899346Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"":"downstairs","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354909025Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354919114Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"client":"0","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354928703Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"client":"1","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354938322Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"client":"2","":"downstairs","session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354948091Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.35495713Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"4710e400-63d2-4dd5-8296-e4c293c9c213 is now active with session: 48e2b089-5d27-4276-993d-39aa0182741a","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354967038Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354975797Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"session_id":"48e2b089-5d27-4276-993d-39aa0182741a"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354984896Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638} {"msg":"Activated sub_volume 4710e400-63d2-4dd5-8296-e4c293c9c213","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.354992905Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.355000964Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.355273553Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"task":"crutest"} 01/30 Write block 323 len 20480 data: 9 10 10 9 9 02/30 Read block 0 len 40960 03/30 Write block 421 len 4096 data: 11 04/30 Write block 373 len 20480 data: 3 4 7 8 6 05/30 Read block 383 len 4096 06/30 Write block 127 len 4096 data: 11 07/30 Read block 596 len 16384 08/30 Flush 09/30 Write block 477 len 8192 data: 11 11 10/30 Write block 128 len 36864 data: 11 11 11 9 9 8 7 9 7 11/30 Read block 222 len 40960 12/30 Write block 534 len 24576 data: 11 11 11 8 10 10 13/30 Read block 49 len 40960 14/30 Read block 222 len 16384 15/30 Flush 16/30 Write block 372 len 40960 data: 5 4 5 8 9 7 6 6 8 7 17/30 Read block 511 len 20480 18/30 Read block 298 len 24576 19/30 Flush 20/30 Write block 411 len 16384 data: 8 9 8 8 21/30 Read block 294 len 12288 22/30 Read block 47 len 16384 23/30 Flush 24/30 Write block 63 len 32768 data: 8 9 9 8 8 8 8 8 25/30 Write block 458 len 4096 data: 9 26/30 Read block 11 len 20480 27/30 Write block 517 len 36864 data: 12 13 12 9 9 9 9 8 8 28/30 Read block 34 len 8192 29/30 Read block 59 len 8192 30/30 Read block 484 len 36864 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Sent false 1001 Acked Read 10 Done Done Sent false 1002 Acked Write 1 Done Done Sent false 1003 Acked Write 5 Done Done Sent false 1004 Acked Read 1 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Write 9 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Read 4 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Read 3 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:40.364134168Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:40.364329386Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1638,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 220 220 220 F F F 1 020-039 120 120 120 224 224 224 F F F 2 040-059 120 120 120 225 225 225 F F F 3 060-079 123 123 123 228 228 228 T T F <--- 4 080-099 117 117 117 218 218 218 F F F 5 100-119 122 122 122 227 227 227 F F F 6 120-139 124 124 111 230 230 205 F F F <--- 7 140-159 120 120 120 224 224 224 F F F 8 160-179 115 115 115 216 216 216 F F F 9 180-199 118 118 118 220 220 220 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 123 123 123 228 228 228 F F F 12 240-259 119 119 119 222 222 222 F F F 13 260-279 113 113 113 208 208 208 F F F 14 280-299 117 117 117 218 218 218 F F F 15 300-319 123 123 123 228 228 228 F F F 16 320-339 124 124 123 229 229 228 F F F <--- 17 340-359 123 123 123 228 228 228 F F F 18 360-379 124 124 117 231 231 218 F F F <--- 19 380-399 124 124 117 231 231 218 F F F <--- 20 400-419 124 124 119 232 232 222 F F F <--- 21 420-439 124 124 116 229 229 217 F F F <--- 22 440-459 118 118 118 219 219 219 T T F <--- 23 460-479 124 124 123 230 230 228 F F F <--- 24 480-499 120 120 120 224 224 224 F F F 25 500-519 118 118 118 219 219 219 T T F <--- 26 520-539 124 124 123 230 230 228 T T F <--- 27 540-559 120 120 120 224 224 224 F F F 28 560-579 123 123 123 228 228 228 F F F 29 580-599 123 123 123 228 228 228 F F F Max gen: 124, Max flush: 232 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 220 220 220 F F F 1 020-039 120 120 120 224 224 224 F F F 2 040-059 120 120 120 225 225 225 F F F 3 060-079 125 125 125 233 233 233 F F F 4 080-099 117 117 117 218 218 218 F F F 5 100-119 122 122 122 227 227 227 F F F 6 120-139 124 124 124 230 230 230 F F F 7 140-159 120 120 120 224 224 224 F F F 8 160-179 115 115 115 216 216 216 F F F 9 180-199 118 118 118 220 220 220 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 123 123 123 228 228 228 F F F 12 240-259 119 119 119 222 222 222 F F F 13 260-279 113 113 113 208 208 208 F F F 14 280-299 117 117 117 218 218 218 F F F 15 300-319 123 123 123 228 228 228 F F F 16 320-339 124 124 124 229 229 229 F F F 17 340-359 123 123 123 228 228 228 F F F 18 360-379 124 124 124 231 231 231 F F F 19 380-399 124 124 124 231 231 231 F F F 20 400-419 124 124 124 232 232 232 F F F 21 420-439 124 124 124 229 229 229 F F F 22 440-459 125 125 125 233 233 233 F F F 23 460-479 124 124 124 230 230 230 F F F 24 480-499 120 120 120 224 224 224 F F F 25 500-519 125 125 125 233 233 233 F F F 26 520-539 125 125 125 233 233 233 F F F 27 540-559 120 120 120 224 224 224 F F F 28 560-579 123 123 123 228 228 228 F F F 29 580-599 123 123 123 228 228 228 F F F Max gen: 125, Max flush: 233 Begin loop 63 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.609398983Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:44.616471497Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.616516202Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.623419356Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.623673927Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.623703033Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.623714052Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.623725591Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646} {"msg":"Crucible 55446923-bf8f-4466-829a-f722fc616e1f has session id: 9f85c6df-4623-432d-9ddb-86ca5dbbdf0f","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624044523Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"Upstairs opts: Upstairs UUID: 55446923-bf8f-4466-829a-f722fc616e1f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.6240737Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"Crucible stats registered with UUID: 55446923-bf8f-4466-829a-f722fc616e1f","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624084859Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:44.624094148Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624240491Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624269017Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624279866Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624396992Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.62441224Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646} {"msg":"55446923-bf8f-4466-829a-f722fc616e1f active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624444607Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624454236Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"client":"0","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624463814Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"client":"1","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624475403Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"client":"2","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624485012Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624495071Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624505Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:36269)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624578721Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37022)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624656262Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"ds_connection connected from Ok(127.0.0.1:34497)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.624680949Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625238154Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"client":"2","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625251642Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"client":"0","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625262441Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"client":"1","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625538899Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625548638Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"[0]R flush_numbers[0..12]: [220, 224, 225, 233, 218, 227, 230, 224, 216, 220, 228, 228]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.62561329Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"downstairs","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"[0]R generation[0..12]: [118, 120, 120, 125, 117, 122, 124, 120, 115, 118, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625624899Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"downstairs","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625636188Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"downstairs","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"[1]R flush_numbers[0..12]: [220, 224, 225, 233, 218, 227, 230, 224, 216, 220, 228, 228]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625649266Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"downstairs","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"[1]R generation[0..12]: [118, 120, 120, 125, 117, 122, 124, 120, 115, 118, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625659995Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"downstairs","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625670914Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"downstairs","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"[2]R flush_numbers[0..12]: [220, 224, 225, 233, 218, 227, 230, 224, 216, 220, 228, 228]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625681462Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"downstairs","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"[2]R generation[0..12]: [118, 120, 120, 125, 117, 122, 124, 120, 115, 118, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625692021Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"downstairs","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.62570283Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"downstairs","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625712219Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"downstairs","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625722018Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"downstairs","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"Next flush: 234","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625731326Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"downstairs","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625740685Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"":"downstairs","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625750294Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625759063Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"client":"0","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625768752Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"client":"1","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625778371Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"client":"2","":"downstairs","session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.6257883Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625798829Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"55446923-bf8f-4466-829a-f722fc616e1f is now active with session: 9f85c6df-4623-432d-9ddb-86ca5dbbdf0f","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625808767Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625817386Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"session_id":"9f85c6df-4623-432d-9ddb-86ca5dbbdf0f"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625826425Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646} {"msg":"Activated sub_volume 55446923-bf8f-4466-829a-f722fc616e1f","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625834544Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.625842833Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.626139769Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"task":"crutest"} 01/30 Write block 177 len 24576 data: 7 12 12 12 11 9 02/30 Write block 581 len 20480 data: 12 11 9 7 8 03/30 Read block 53 len 32768 04/30 Flush 05/30 Write block 498 len 32768 data: 9 11 11 9 6 6 8 11 06/30 Write block 249 len 36864 data: 9 11 12 12 11 10 9 10 10 07/30 Flush 08/30 Write block 377 len 12288 data: 8 7 7 09/30 Read block 106 len 16384 10/30 Read block 363 len 8192 11/30 Read block 398 len 20480 12/30 Read block 418 len 12288 13/30 Write block 301 len 40960 data: 14 13 15 12 16 16 16 13 12 12 14/30 Flush 15/30 Write block 87 len 40960 data: 12 12 15 14 13 14 15 15 14 11 16/30 Read block 299 len 16384 17/30 Write block 428 len 20480 data: 9 9 9 11 11 18/30 Write block 237 len 36864 data: 12 14 15 13 16 17 14 14 12 19/30 Read block 164 len 24576 20/30 Read block 27 len 16384 21/30 Read block 445 len 24576 22/30 Read block 584 len 24576 23/30 Write block 264 len 24576 data: 7 6 8 11 12 14 24/30 Write block 520 len 20480 data: 10 10 10 10 9 25/30 Read block 303 len 8192 26/30 Write block 593 len 24576 data: 9 7 9 8 8 7 27/30 Write block 90 len 20480 data: 15 14 15 16 16 28/30 Write block 434 len 16384 data: 12 10 10 6 29/30 Write block 393 len 40960 data: 9 10 10 10 9 11 11 12 10 7 30/30 Read block 555 len 40960 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Read 3 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Write 10 Sent Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 1 29 Done 2 30 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:44.635215839Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:44.635392039Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1646,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 220 220 220 F F F 1 020-039 120 120 120 224 224 224 F F F 2 040-059 120 120 120 225 225 225 F F F 3 060-079 125 125 125 233 233 233 F F F 4 080-099 117 117 117 218 218 218 F T T <--- 5 100-119 122 122 122 227 227 227 F F F 6 120-139 124 124 124 230 230 230 F F F 7 140-159 120 120 120 224 224 224 F F F 8 160-179 115 126 126 216 234 234 T F F <--- 9 180-199 118 126 126 220 234 234 T F F <--- 10 200-219 123 123 123 228 228 228 F F F 11 220-239 123 123 123 228 228 228 F T T <--- 12 240-259 119 126 126 222 235 235 F T T <--- 13 260-279 113 113 113 208 208 208 F T T <--- 14 280-299 117 117 117 218 218 218 F F F 15 300-319 123 126 126 228 236 236 F F F <--- 16 320-339 124 124 124 229 229 229 F F F 17 340-359 123 123 123 228 228 228 F F F 18 360-379 124 126 126 231 236 236 F F F <--- 19 380-399 124 124 124 231 231 231 F T T <--- 20 400-419 124 124 124 232 232 232 F T T <--- 21 420-439 124 124 124 229 229 229 F T T <--- 22 440-459 125 125 125 233 233 233 F F F 23 460-479 124 124 124 230 230 230 F F F 24 480-499 120 126 126 224 235 235 F F F <--- 25 500-519 125 126 126 233 235 235 F F F <--- 26 520-539 125 125 125 233 233 233 F T T <--- 27 540-559 120 120 120 224 224 224 F F F 28 560-579 123 123 123 228 228 228 F F F 29 580-599 123 126 126 228 234 234 T T T <--- Max gen: 126, Max flush: 236 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 220 220 220 F F F 1 020-039 120 120 120 224 224 224 F F F 2 040-059 120 120 120 225 225 225 F F F 3 060-079 125 125 125 233 233 233 F F F 4 080-099 127 127 127 237 237 237 F F F 5 100-119 122 122 122 227 227 227 F F F 6 120-139 124 124 124 230 230 230 F F F 7 140-159 120 120 120 224 224 224 F F F 8 160-179 126 126 126 234 234 234 F F F 9 180-199 126 126 126 234 234 234 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 127 127 127 237 237 237 F F F 12 240-259 127 127 127 237 237 237 F F F 13 260-279 127 127 127 237 237 237 F F F 14 280-299 117 117 117 218 218 218 F F F 15 300-319 126 126 126 236 236 236 F F F 16 320-339 124 124 124 229 229 229 F F F 17 340-359 123 123 123 228 228 228 F F F 18 360-379 126 126 126 236 236 236 F F F 19 380-399 127 127 127 237 237 237 F F F 20 400-419 127 127 127 237 237 237 F F F 21 420-439 127 127 127 237 237 237 F F F 22 440-459 125 125 125 233 233 233 F F F 23 460-479 124 124 124 230 230 230 F F F 24 480-499 126 126 126 235 235 235 F F F 25 500-519 126 126 126 235 235 235 F F F 26 520-539 127 127 127 237 237 237 F F F 27 540-559 120 120 120 224 224 224 F F F 28 560-579 123 123 123 228 228 228 F F F 29 580-599 127 127 127 237 237 237 F F F Max gen: 127, Max flush: 237 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.900487038Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.908754073Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909050579Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909084055Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909093434Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909105042Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654} {"msg":"Crucible ec28a12e-8fe2-4a63-b657-a7187e254b87 has session id: 7feaa1f3-f145-4d00-95ea-0030a8198d19","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909401418Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"Upstairs opts: Upstairs UUID: ec28a12e-8fe2-4a63-b657-a7187e254b87, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909424865Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"Crucible stats registered with UUID: ec28a12e-8fe2-4a63-b657-a7187e254b87","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909437493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:48.909446702Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909595495Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909624622Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.90963546Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909747937Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909771434Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654} {"msg":"ec28a12e-8fe2-4a63-b657-a7187e254b87 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909783433Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909792472Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"client":"0","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909802501Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"client":"1","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.90981228Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"client":"2","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909822078Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909832247Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.909842336Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"ds_connection connected from Ok(127.0.0.1:61098)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.90989109Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"ds_connection connected from Ok(127.0.0.1:53205)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.910018486Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"ds_connection connected from Ok(127.0.0.1:43679)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.910032234Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.910605927Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"client":"0","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.910628824Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"client":"2","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.910640163Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"client":"1","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.910930679Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.910953436Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"[0]R flush_numbers[0..12]: [220, 224, 225, 233, 237, 227, 230, 224, 234, 234, 228, 237]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.910964185Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"downstairs","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"[0]R generation[0..12]: [118, 120, 120, 125, 127, 122, 124, 120, 126, 126, 123, 127]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.910975344Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"downstairs","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.910986423Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"downstairs","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"[1]R flush_numbers[0..12]: [220, 224, 225, 233, 237, 227, 230, 224, 234, 234, 228, 237]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.910997011Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"downstairs","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"[1]R generation[0..12]: [118, 120, 120, 125, 127, 122, 124, 120, 126, 126, 123, 127]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.91100766Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"downstairs","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911019009Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"downstairs","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"[2]R flush_numbers[0..12]: [220, 224, 225, 233, 237, 227, 230, 224, 234, 234, 228, 237]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911029648Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"downstairs","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"[2]R generation[0..12]: [118, 120, 120, 125, 127, 122, 124, 120, 126, 126, 123, 127]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911042916Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"downstairs","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911053925Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"downstairs","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911063284Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"downstairs","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911073202Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"downstairs","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911082791Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"downstairs","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.91109223Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"":"downstairs","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911107008Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911115707Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"client":"0","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911125466Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"client":"1","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911135075Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"client":"2","":"downstairs","session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911144864Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911154063Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"ec28a12e-8fe2-4a63-b657-a7187e254b87 is now active with session: 7feaa1f3-f145-4d00-95ea-0030a8198d19","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911164022Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911172801Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"session_id":"7feaa1f3-f145-4d00-95ea-0030a8198d19"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.91118196Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 } {"msg":"Activated sub_volume ec28a12e-8fe2-4a63-b657-a7187e254b87","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911195878Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911205237Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.911447779Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"task":"crutest"} 01/30 Write block 452 len 4096 data: 4 02/30 Flush 03/30 Write block 396 len 12288 data: 11 10 12 04/30 Write block 140 len 24576 data: 11 10 12 12 12 13 05/30 Flush 06/30 Write block 571 len 36864 data: 11 12 11 12 12 12 13 10 13 07/30 Read block 137 len 12288 08/30 Read block 581 len 28672 09/30 Write block 438 len 20480 data: 6 8 8 11 12 10/30 Read block 345 len 32768 11/30 Write block 273 len 40960 data: 12 12 14 14 12 13 14 12 9 9 12/30 Read block 35 len 28672 13/30 Write block 476 len 4096 data: 11 14/30 Write block 5 len 32768 data: 12 13 12 13 10 11 10 11 15/30 Write block 28 len 32768 data: 16 17 18 19 18 16 13 11 16/30 Read block 551 len 24576 17/30 Write block 311 len 12288 data: 6 7 8 18/30 Write block 441 len 36864 data: 12 13 10 11 12 10 9 8 7 19/30 Read block 497 len 8192 20/30 Read block 437 len 8192 21/30 Read block 390 len 16384 22/30 Write block 371 len 16384 data: 6 6 5 6 23/30 Read block 151 len 40960 24/30 Write block 292 len 24576 data: 7 11 13 12 12 13 25/30 Write block 178 len 28672 data: 13 13 13 12 10 11 9 26/30 Read block 542 len 16384 27/30 Write block 36 len 16384 data: 9 8 8 8 28/30 Write block 105 len 16384 data: 6 11 9 9 29/30 Read block 107 len 8192 30/30 Write block 530 len 16384 data: 10 10 11 12 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 3 Done Sent Done false 1003 Acked Write 6 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Write 10 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Write 8 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 27 0 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1001)) Some(JobId(1004)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:48.919813772Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:48.919984962Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1654,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 118 118 118 220 220 220 T F T <--- 1 020-039 120 120 120 224 224 224 T F T <--- 2 040-059 120 120 120 225 225 225 F F F 3 060-079 125 125 125 233 233 233 F F F 4 080-099 127 127 127 237 237 237 F F F 5 100-119 122 122 122 227 227 227 T F T <--- 6 120-139 124 124 124 230 230 230 F F F 7 140-159 128 120 128 239 224 239 F F F <--- 8 160-179 126 126 126 234 234 234 T F T <--- 9 180-199 126 126 126 234 234 234 T F T <--- 10 200-219 123 123 123 228 228 228 F F F 11 220-239 127 127 127 237 237 237 F F F 12 240-259 127 127 127 237 237 237 F F F 13 260-279 127 127 127 237 237 237 T F T <--- 14 280-299 117 117 117 218 218 218 T F T <--- 15 300-319 126 126 126 236 236 236 T F T <--- 16 320-339 124 124 124 229 229 229 F F F 17 340-359 123 123 123 228 228 228 F F F 18 360-379 126 126 126 236 236 236 T F T <--- 19 380-399 128 127 128 239 237 239 F F F <--- 20 400-419 127 127 127 237 237 237 F F F 21 420-439 127 127 127 237 237 237 T F T <--- 22 440-459 128 128 128 238 238 238 T F T <--- 23 460-479 124 124 124 230 230 230 T F T <--- 24 480-499 126 126 126 235 235 235 F F F 25 500-519 126 126 126 235 235 235 F F F 26 520-539 127 127 127 237 237 237 T F T <--- 27 540-559 120 120 120 224 224 224 F F F 28 560-579 123 123 123 228 228 228 T F T <--- 29 580-599 127 127 127 237 237 237 F F F Max gen: 128, Max flush: 239 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 240 240 240 F F F 1 020-039 129 129 129 240 240 240 F F F 2 040-059 120 120 120 225 225 225 F F F 3 060-079 125 125 125 233 233 233 F F F 4 080-099 127 127 127 237 237 237 F F F 5 100-119 129 129 129 240 240 240 F F F 6 120-139 124 124 124 230 230 230 F F F 7 140-159 128 128 128 239 239 239 F F F 8 160-179 129 129 129 240 240 240 F F F 9 180-199 129 129 129 240 240 240 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 127 127 127 237 237 237 F F F 12 240-259 127 127 127 237 237 237 F F F 13 260-279 129 129 129 240 240 240 F F F 14 280-299 129 129 129 240 240 240 F F F 15 300-319 129 129 129 240 240 240 F F F 16 320-339 124 124 124 229 229 229 F F F 17 340-359 123 123 123 228 228 228 F F F 18 360-379 129 129 129 240 240 240 F F F 19 380-399 128 128 128 239 239 239 F F F 20 400-419 127 127 127 237 237 237 F F F 21 420-439 129 129 129 240 240 240 F F F 22 440-459 129 129 129 240 240 240 F F F 23 460-479 129 129 129 240 240 240 F F F 24 480-499 126 126 126 235 235 235 F F F 25 500-519 126 126 126 235 235 235 F F F 26 520-539 129 129 129 240 240 240 F F F 27 540-559 120 120 120 224 224 224 F F F 28 560-579 129 129 129 240 240 240 F F F 29 580-599 127 127 127 237 237 237 F F F Max gen: 129, Max flush: 240 Begin loop 65 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.180589856Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:53.187708425Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.187763669Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.194743044Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.194994845Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195020872Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195029531Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195041069Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662} {"msg":"Crucible b5c01d5e-e430-46aa-a458-19de9069eef3 has session id: 40db3ca0-5a4b-4990-b3eb-c864f43b9e78","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195404177Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"Upstairs opts: Upstairs UUID: b5c01d5e-e430-46aa-a458-19de9069eef3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195432604Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"Crucible stats registered with UUID: b5c01d5e-e430-46aa-a458-19de9069eef3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195443632Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:53.195452911Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195574667Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195589845Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195599814Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195745657Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195761055Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662} {"msg":"b5c01d5e-e430-46aa-a458-19de9069eef3 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195792612Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.19580215Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"client":"0","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195812019Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"client":"1","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195821858Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"client":"2","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195831437Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195841516Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195851425Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"ds_connection connected from Ok(127.0.0.1:59768)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.19589261Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"ds_connection connected from Ok(127.0.0.1:52723)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195939724Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"ds_connection connected from Ok(127.0.0.1:43583)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.195953853Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.196594678Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"client":"2","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.196633863Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"client":"1","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.196645772Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"client":"0","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.196903262Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.19692273Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"[0]R flush_numbers[0..12]: [240, 240, 225, 233, 237, 240, 230, 239, 240, 240, 228, 237]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.196978983Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"downstairs","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"[0]R generation[0..12]: [129, 129, 120, 125, 127, 129, 124, 128, 129, 129, 123, 127]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.196990552Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"downstairs","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.19700162Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"downstairs","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"[1]R flush_numbers[0..12]: [240, 240, 225, 233, 237, 240, 230, 239, 240, 240, 228, 237]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197012609Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"downstairs","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"[1]R generation[0..12]: [129, 129, 120, 125, 127, 129, 124, 128, 129, 129, 123, 127]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197023318Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"downstairs","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197034117Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"downstairs","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"[2]R flush_numbers[0..12]: [240, 240, 225, 233, 237, 240, 230, 239, 240, 240, 228, 237]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197044595Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"downstairs","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"[2]R generation[0..12]: [129, 129, 120, 125, 127, 129, 124, 128, 129, 129, 123, 127]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197055244Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"downstairs","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197066053Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"downstairs","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197075462Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"downstairs","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.19708691Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"downstairs","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197096389Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"downstairs","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197105808Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"":"downstairs","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197115467Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197124206Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"client":"0","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197133975Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"client":"1","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197143554Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"client":"2","":"downstairs","session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197153473Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197162582Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"b5c01d5e-e430-46aa-a458-19de9069eef3 is now active with session: 40db3ca0-5a4b-4990-b3eb-c864f43b9e78","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.19717247Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197181169Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"session_id":"40db3ca0-5a4b-4990-b3eb-c864f43b9e78"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197190268Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662} {"msg":"Activated sub_volume b5c01d5e-e430-46aa-a458-19de9069eef3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197198367Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197206387Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.197539518Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"task":"crutest"} 01/30 Write block 377 len 20480 data: 9 8 8 9 8 02/30 Write block 129 len 20480 data: 12 12 10 10 9 03/30 Read block 535 len 36864 04/30 Read block 506 len 36864 05/30 Read block 279 len 28672 06/30 Write block 54 len 36864 data: 13 13 15 11 11 8 8 9 8 07/30 Write block 426 len 40960 data: 8 10 10 10 10 12 12 12 13 11 08/30 Read block 296 len 20480 09/30 Write block 446 len 4096 data: 11 10/30 Read block 154 len 12288 11/30 Read block 483 len 16384 12/30 Write block 220 len 32768 data: 6 6 8 11 11 11 11 12 13/30 Write block 118 len 28672 data: 13 13 12 12 12 13 13 14/30 Write block 38 len 28672 data: 9 9 11 10 9 8 9 15/30 Read block 163 len 12288 16/30 Read block 6 len 4096 17/30 Read block 535 len 24576 18/30 Write block 503 len 4096 data: 7 19/30 Flush 20/30 Write block 533 len 32768 data: 13 12 12 12 9 11 11 9 21/30 Read block 326 len 24576 22/30 Flush 23/30 Write block 237 len 4096 data: 13 24/30 Write block 438 len 8192 data: 7 9 25/30 Read block 541 len 16384 26/30 Write block 281 len 40960 data: 10 10 7 7 7 6 7 6 7 8 27/30 Write block 169 len 12288 data: 9 6 8 28/30 Read block 540 len 32768 29/30 Write block 257 len 4096 data: 11 30/30 Read block 197 len 12288 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 5 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Read 9 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Write 9 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Read 4 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Write 2 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 3 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 1 30 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:53.206060613Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:53.206247441Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1662,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 240 240 240 F F F 1 020-039 129 130 130 240 241 241 F F F <--- 2 040-059 120 130 130 225 241 241 F F F <--- 3 060-079 125 130 130 233 241 241 F F F <--- 4 080-099 127 127 127 237 237 237 F F F 5 100-119 129 130 130 240 241 241 F F F <--- 6 120-139 124 130 130 230 241 241 F F F <--- 7 140-159 128 128 128 239 239 239 F F F 8 160-179 129 129 129 240 240 240 F T T <--- 9 180-199 129 129 129 240 240 240 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 127 130 130 237 241 241 F T T <--- 12 240-259 127 127 127 237 237 237 F T T <--- 13 260-279 129 129 129 240 240 240 F F F 14 280-299 129 129 129 240 240 240 F T T <--- 15 300-319 129 129 129 240 240 240 F F F 16 320-339 124 124 124 229 229 229 F F F 17 340-359 123 123 123 228 228 228 F F F 18 360-379 129 130 130 240 241 241 T F F <--- 19 380-399 128 130 130 239 241 241 T F F <--- 20 400-419 127 127 127 237 237 237 F F F 21 420-439 129 130 130 240 241 241 F T T <--- 22 440-459 129 130 130 240 241 241 F F F <--- 23 460-479 129 129 129 240 240 240 F F F 24 480-499 126 126 126 235 235 235 F F F 25 500-519 126 130 130 235 241 241 F F F <--- 26 520-539 129 130 130 240 242 242 F F F <--- 27 540-559 120 130 130 224 242 242 F F F <--- 28 560-579 129 129 129 240 240 240 F F F 29 580-599 127 127 127 237 237 237 F F F Max gen: 130, Max flush: 242 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 129 129 129 240 240 240 F F F 1 020-039 130 130 130 241 241 241 F F F 2 040-059 130 130 130 241 241 241 F F F 3 060-079 130 130 130 241 241 241 F F F 4 080-099 127 127 127 237 237 237 F F F 5 100-119 130 130 130 241 241 241 F F F 6 120-139 130 130 130 241 241 241 F F F 7 140-159 128 128 128 239 239 239 F F F 8 160-179 131 131 131 243 243 243 F F F 9 180-199 129 129 129 240 240 240 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 131 131 131 243 243 243 F F F 12 240-259 131 131 131 243 243 243 F F F 13 260-279 129 129 129 240 240 240 F F F 14 280-299 131 131 131 243 243 243 F F F 15 300-319 129 129 129 240 240 240 F F F 16 320-339 124 124 124 229 229 229 F F F 17 340-359 123 123 123 228 228 228 F F F 18 360-379 130 130 130 241 241 241 F F F 19 380-399 130 130 130 241 241 241 F F F 20 400-419 127 127 127 237 237 237 F F F 21 420-439 131 131 131 243 243 243 F F F 22 440-459 130 130 130 241 241 241 F F F 23 460-479 129 129 129 240 240 240 F F F 24 480-499 126 126 126 235 235 235 F F F 25 500-519 130 130 130 241 241 241 F F F 26 520-539 130 130 130 242 242 242 F F F 27 540-559 130 130 130 242 242 242 F F F 28 560-579 129 129 129 240 240 240 F F F 29 580-599 127 127 127 237 237 237 F F F Max gen: 131, Max flush: 243 Begin loop 66 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.469901129Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.478265763Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.4785499Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.478574497Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.478582986Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.478594405Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670} {"msg":"Crucible c47e97a0-3124-416e-8cfe-bfa826fda109 has session id: db486c6c-dae8-4b9f-af83-6b5510915b9b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.478930705Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"Upstairs opts: Upstairs UUID: c47e97a0-3124-416e-8cfe-bfa826fda109, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.478959532Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"Crucible stats registered with UUID: c47e97a0-3124-416e-8cfe-bfa826fda109","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.478970341Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:57.47897948Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.479110744Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.479138551Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.47914952Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.479269056Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.479297682Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670} {"msg":"c47e97a0-3124-416e-8cfe-bfa826fda109 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.47931972Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.479331958Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"client":"0","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.479342437Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"client":"1","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.479352286Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"client":"2","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.479361835Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.479371884Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.479381763Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50124)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.479428967Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64624)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.479446485Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56737)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.479528645Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480076782Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"client":"1","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.48009284Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"client":"0","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480103628Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"client":"2","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480387905Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480427231Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"[0]R flush_numbers[0..12]: [240, 241, 241, 241, 237, 241, 241, 239, 243, 240, 228, 243]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480503022Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"downstairs","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"[0]R generation[0..12]: [129, 130, 130, 130, 127, 130, 130, 128, 131, 129, 123, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.48051573Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"downstairs","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480526779Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"downstairs","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"[1]R flush_numbers[0..12]: [240, 241, 241, 241, 237, 241, 241, 239, 243, 240, 228, 243]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480537558Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"downstairs","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"[1]R generation[0..12]: [129, 130, 130, 130, 127, 130, 130, 128, 131, 129, 123, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480548096Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"downstairs","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480558955Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"downstairs","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"[2]R flush_numbers[0..12]: [240, 241, 241, 241, 237, 241, 241, 239, 243, 240, 228, 243]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480569594Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"downstairs","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"[2]R generation[0..12]: [129, 130, 130, 130, 127, 130, 130, 128, 131, 129, 123, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480580043Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"downstairs","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480591021Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"downstairs","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.48060057Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"downstairs","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480610409Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"downstairs","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480619758Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"downstairs","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480629157Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"":"downstairs","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480639196Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480648065Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"client":"0","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480659513Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"client":"1","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480669532Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"client":"2","":"downstairs","session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480679641Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.48068888Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"c47e97a0-3124-416e-8cfe-bfa826fda109 is now active with session: db486c6c-dae8-4b9f-af83-6b5510915b9b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480698829Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480707578Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"session_id":"db486c6c-dae8-4b9f-af83-6b5510915b9b"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480716737Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670} {"msg":"Activated sub_volume c47e97a0-3124-416e-8cfe-bfa826fda109","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480724726Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.480732845Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.481005123Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"task":"crutest"} 01/30 Write block 185 len 36864 data: 9 8 5 7 12 13 14 11 10 02/30 Read block 578 len 40960 03/30 Read block 333 len 40960 04/30 Read block 438 len 24576 05/30 Read block 327 len 24576 06/30 Write block 345 len 36864 data: 6 8 8 9 11 10 14 15 15 07/30 Read block 290 len 32768 08/30 Read block 525 len 36864 09/30 Read block 232 len 36864 10/30 Read block 168 len 32768 11/30 Read block 559 len 36864 12/30 Read block 274 len 36864 13/30 Write block 1 len 8192 data: 5 7 14/30 Read block 155 len 12288 15/30 Read block 476 len 12288 16/30 Read block 141 len 28672 17/30 Write block 309 len 12288 data: 13 13 7 18/30 Write block 309 len 16384 data: 14 14 8 8 19/30 Flush 20/30 Write block 586 len 40960 data: 7 7 6 10 9 8 9 10 8 10 21/30 Read block 122 len 32768 22/30 Write block 9 len 12288 data: 11 12 11 23/30 Read block 290 len 16384 24/30 Read block 556 len 16384 25/30 Write block 542 len 8192 data: 11 12 26/30 Write block 190 len 36864 data: 14 15 12 11 11 11 10 10 11 27/30 Write block 335 len 4096 data: 12 28/30 Write block 2 len 8192 data: 8 8 29/30 Write block 446 len 16384 data: 12 10 9 8 30/30 Read block 498 len 32768 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 6 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Write 3 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 26 26 Done 30 30 4 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:43:57.490282Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:43:57.490475478Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1670,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 129 244 244 240 T T F <--- 1 020-039 130 130 130 241 241 241 F F F 2 040-059 130 130 130 241 241 241 F F F 3 060-079 130 130 130 241 241 241 F F F 4 080-099 127 127 127 237 237 237 F F F 5 100-119 130 130 130 241 241 241 F F F 6 120-139 130 130 130 241 241 241 F F F 7 140-159 128 128 128 239 239 239 F F F 8 160-179 131 131 131 243 243 243 F F F 9 180-199 132 132 129 244 244 240 T T T <--- 10 200-219 123 123 123 228 228 228 F F F 11 220-239 131 131 131 243 243 243 F F F 12 240-259 131 131 131 243 243 243 F F F 13 260-279 129 129 129 240 240 240 F F F 14 280-299 131 131 131 243 243 243 F F F 15 300-319 132 132 129 244 244 240 F F F <--- 16 320-339 124 124 124 229 229 229 T T F <--- 17 340-359 132 132 123 244 244 228 F F F <--- 18 360-379 130 130 130 241 241 241 F F F 19 380-399 130 130 130 241 241 241 F F F 20 400-419 127 127 127 237 237 237 F F F 21 420-439 131 131 131 243 243 243 F F F 22 440-459 130 130 130 241 241 241 T T F <--- 23 460-479 129 129 129 240 240 240 F F F 24 480-499 126 126 126 235 235 235 F F F 25 500-519 130 130 130 241 241 241 F F F 26 520-539 130 130 130 242 242 242 F F F 27 540-559 130 130 130 242 242 242 T T F <--- 28 560-579 129 129 129 240 240 240 F F F 29 580-599 127 127 127 237 237 237 T T F <--- Max gen: 132, Max flush: 244 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 245 245 245 F F F 1 020-039 130 130 130 241 241 241 F F F 2 040-059 130 130 130 241 241 241 F F F 3 060-079 130 130 130 241 241 241 F F F 4 080-099 127 127 127 237 237 237 F F F 5 100-119 130 130 130 241 241 241 F F F 6 120-139 130 130 130 241 241 241 F F F 7 140-159 128 128 128 239 239 239 F F F 8 160-179 131 131 131 243 243 243 F F F 9 180-199 133 133 133 245 245 245 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 131 131 131 243 243 243 F F F 12 240-259 131 131 131 243 243 243 F F F 13 260-279 129 129 129 240 240 240 F F F 14 280-299 131 131 131 243 243 243 F F F 15 300-319 132 132 132 244 244 244 F F F 16 320-339 133 133 133 245 245 245 F F F 17 340-359 132 132 132 244 244 244 F F F 18 360-379 130 130 130 241 241 241 F F F 19 380-399 130 130 130 241 241 241 F F F 20 400-419 127 127 127 237 237 237 F F F 21 420-439 131 131 131 243 243 243 F F F 22 440-459 133 133 133 245 245 245 F F F 23 460-479 129 129 129 240 240 240 F F F 24 480-499 126 126 126 235 235 235 F F F 25 500-519 130 130 130 241 241 241 F F F 26 520-539 130 130 130 242 242 242 F F F 27 540-559 133 133 133 245 245 245 F F F 28 560-579 129 129 129 240 240 240 F F F 29 580-599 133 133 133 245 245 245 F F F Max gen: 133, Max flush: 245 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.734424716Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.742625689Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.742901666Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.742931713Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.742940462Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.7429519Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678} {"msg":"Crucible 4259994a-257f-447c-a7d1-c046129bda3d has session id: eb9554a4-a51a-4cfa-a421-7c77f3937524","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743286091Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"Upstairs opts: Upstairs UUID: 4259994a-257f-447c-a7d1-c046129bda3d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743309339Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"Crucible stats registered with UUID: 4259994a-257f-447c-a7d1-c046129bda3d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743319348Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:01.743328287Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.74346975Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743497527Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743508506Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743657488Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743669677Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678} {"msg":"4259994a-257f-447c-a7d1-c046129bda3d active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743681885Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743690734Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"client":"0","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743700393Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"client":"1","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743710022Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"client":"2","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743719631Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743746948Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743757316Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"ds_connection connected from Ok(127.0.0.1:45580)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743795682Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"ds_connection connected from Ok(127.0.0.1:37791)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.74381238Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"ds_connection connected from Ok(127.0.0.1:35738)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.743823209Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744386873Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"client":"2","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744471043Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"client":"0","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744556113Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"client":"1","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744691337Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744810404Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"[0]R flush_numbers[0..12]: [245, 241, 241, 241, 237, 241, 241, 239, 243, 245, 228, 243]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744892084Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"downstairs","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"[0]R generation[0..12]: [133, 130, 130, 130, 127, 130, 130, 128, 131, 133, 123, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744903823Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"downstairs","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744915141Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"downstairs","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"[1]R flush_numbers[0..12]: [245, 241, 241, 241, 237, 241, 241, 239, 243, 245, 228, 243]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.74492579Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"downstairs","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"[1]R generation[0..12]: [133, 130, 130, 130, 127, 130, 130, 128, 131, 133, 123, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744936539Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"downstairs","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744947458Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"downstairs","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"[2]R flush_numbers[0..12]: [245, 241, 241, 241, 237, 241, 241, 239, 243, 245, 228, 243]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744960526Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"downstairs","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"[2]R generation[0..12]: [133, 130, 130, 130, 127, 130, 130, 128, 131, 133, 123, 131]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744970995Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"downstairs","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744981934Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"downstairs","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.744991312Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"downstairs","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.745001101Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"downstairs","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.74501054Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"downstairs","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.745019929Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"":"downstairs","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.745029778Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.745038707Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"client":"0","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.745048276Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"client":"1","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.745058045Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"client":"2","":"downstairs","session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.745067864Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.745077122Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"4259994a-257f-447c-a7d1-c046129bda3d is now active with session: eb9554a4-a51a-4cfa-a421-7c77f3937524","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.745086891Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.74509564Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"session_id":"eb9554a4-a51a-4cfa-a421-7c77f3937524"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.745105959Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678} {"msg":"Activated sub_volume 4259994a-257f-447c-a7d1-c046129bda3d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.745113918Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.745121997Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.745412403Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"task":"crutest"} 01/30 Write block 74 len 20480 data: 9 9 9 7 10 02/30 Read block 470 len 40960 03/30 Read block 373 len 12288 04/30 Read block 199 len 12288 05/30 Read block 238 len 36864 06/30 Write block 486 len 36864 data: 9 9 12 10 12 12 16 15 16 07/30 Write block 555 len 40960 data: 16 14 11 11 11 8 8 9 10 12 08/30 Write block 325 len 36864 data: 11 10 10 11 11 13 11 13 14 09/30 Read block 310 len 24576 10/30 Write block 56 len 20480 data: 16 12 12 9 9 11/30 Write block 244 len 4096 data: 15 12/30 Read block 177 len 28672 13/30 Read block 31 len 4096 14/30 Write block 38 len 4096 data: 10 15/30 Write block 502 len 24576 data: 7 8 9 12 12 12 16/30 Write block 45 len 12288 data: 11 12 11 17/30 Write block 233 len 24576 data: 8 8 12 11 14 15 18/30 Write block 394 len 28672 data: 11 11 12 11 13 12 13 19/30 Read block 215 len 24576 20/30 Read block 459 len 12288 21/30 Write block 164 len 36864 data: 9 10 10 8 10 10 7 9 9 22/30 Read block 401 len 28672 23/30 Read block 413 len 28672 24/30 Read block 238 len 8192 25/30 Read block 317 len 28672 26/30 Read block 246 len 24576 27/30 Read block 3 len 40960 28/30 Read block 343 len 20480 29/30 Write block 503 len 4096 data: 9 30/30 Read block 261 len 32768 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 10 Done Sent Done false 1002 Acked Read 3 Done Sent Done false 1003 Acked Read 3 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 0 30 Done 29 1 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:01.753793365Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:01.753960475Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1678,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 245 245 245 F F F 1 020-039 130 130 130 241 241 241 T F T <--- 2 040-059 130 130 130 241 241 241 T F T <--- 3 060-079 130 130 130 241 241 241 T T T 4 080-099 127 127 127 237 237 237 F F F 5 100-119 130 130 130 241 241 241 F F F 6 120-139 130 130 130 241 241 241 F F F 7 140-159 128 128 128 239 239 239 F F F 8 160-179 131 131 131 243 243 243 T F T <--- 9 180-199 133 133 133 245 245 245 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 131 131 131 243 243 243 T F T <--- 12 240-259 131 131 131 243 243 243 T F T <--- 13 260-279 129 129 129 240 240 240 F F F 14 280-299 131 131 131 243 243 243 F F F 15 300-319 132 132 132 244 244 244 F F F 16 320-339 133 133 133 245 245 245 T F T <--- 17 340-359 132 132 132 244 244 244 F F F 18 360-379 130 130 130 241 241 241 F F F 19 380-399 130 130 130 241 241 241 T F T <--- 20 400-419 127 127 127 237 237 237 T F T <--- 21 420-439 131 131 131 243 243 243 F F F 22 440-459 133 133 133 245 245 245 F F F 23 460-479 129 129 129 240 240 240 F F F 24 480-499 126 126 126 235 235 235 T F T <--- 25 500-519 130 130 130 241 241 241 T F T <--- 26 520-539 130 130 130 242 242 242 F F F 27 540-559 133 133 133 245 245 245 T F T <--- 28 560-579 129 129 129 240 240 240 T F T <--- 29 580-599 133 133 133 245 245 245 F F F Max gen: 133, Max flush: 245 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 245 245 245 F F F 1 020-039 134 134 134 246 246 246 F F F 2 040-059 134 134 134 246 246 246 F F F 3 060-079 134 134 134 246 246 246 F F F 4 080-099 127 127 127 237 237 237 F F F 5 100-119 130 130 130 241 241 241 F F F 6 120-139 130 130 130 241 241 241 F F F 7 140-159 128 128 128 239 239 239 F F F 8 160-179 134 134 134 246 246 246 F F F 9 180-199 133 133 133 245 245 245 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 134 134 134 246 246 246 F F F 12 240-259 134 134 134 246 246 246 F F F 13 260-279 129 129 129 240 240 240 F F F 14 280-299 131 131 131 243 243 243 F F F 15 300-319 132 132 132 244 244 244 F F F 16 320-339 134 134 134 246 246 246 F F F 17 340-359 132 132 132 244 244 244 F F F 18 360-379 130 130 130 241 241 241 F F F 19 380-399 134 134 134 246 246 246 F F F 20 400-419 134 134 134 246 246 246 F F F 21 420-439 131 131 131 243 243 243 F F F 22 440-459 133 133 133 245 245 245 F F F 23 460-479 129 129 129 240 240 240 F F F 24 480-499 134 134 134 246 246 246 F F F 25 500-519 134 134 134 246 246 246 F F F 26 520-539 130 130 130 242 242 242 F F F 27 540-559 134 134 134 246 246 246 F F F 28 560-579 134 134 134 246 246 246 F F F 29 580-599 133 133 133 245 245 245 F F F Max gen: 134, Max flush: 246 Begin loop 68 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.008757447Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.016904076Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.017185183Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.017219399Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.017228708Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.017240357Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686} {"msg":"Crucible 4c4ce03e-ad2b-4d6c-a016-5dcddd16f92c has session id: 4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.017602514Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"Upstairs opts: Upstairs UUID: 4c4ce03e-ad2b-4d6c-a016-5dcddd16f92c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.017625992Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"Crucible stats registered with UUID: 4c4ce03e-ad2b-4d6c-a016-5dcddd16f92c","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.01763684Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:06.017646449Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.017790562Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.017821549Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.017833637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.017972661Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.017988259Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686} {"msg":"4c4ce03e-ad2b-4d6c-a016-5dcddd16f92c active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.018023125Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.018032794Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"client":"0","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.018042493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"client":"1","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.018052092Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"client":"2","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.018061561Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.01807159Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.018081798Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:47037)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.018178997Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:47291)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.01832487Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:54054)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.018349487Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.018856438Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"client":"2","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.018898943Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"client":"1","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.01892409Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"client":"0","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.01918551Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019195858Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"[0]R flush_numbers[0..12]: [245, 246, 246, 246, 237, 241, 241, 239, 246, 245, 228, 246]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019209317Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"downstairs","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"[0]R generation[0..12]: [133, 134, 134, 134, 127, 130, 130, 128, 134, 133, 123, 134]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019220056Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"downstairs","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019230904Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"downstairs","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"[1]R flush_numbers[0..12]: [245, 246, 246, 246, 237, 241, 241, 239, 246, 245, 228, 246]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019241513Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"downstairs","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"[1]R generation[0..12]: [133, 134, 134, 134, 127, 130, 130, 128, 134, 133, 123, 134]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019251942Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"downstairs","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019262991Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"downstairs","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"[2]R flush_numbers[0..12]: [245, 246, 246, 246, 237, 241, 241, 239, 246, 245, 228, 246]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019273489Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"downstairs","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"[2]R generation[0..12]: [133, 134, 134, 134, 127, 130, 130, 128, 134, 133, 123, 134]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019284018Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"downstairs","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019294847Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"downstairs","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"Max found gen is 135","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019304546Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"downstairs","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"Generation requested: 136 >= found:135","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019316654Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"downstairs","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019325953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"downstairs","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019335422Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"":"downstairs","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019345361Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.01935429Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"client":"0","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019363949Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"client":"1","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019373608Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"client":"2","":"downstairs","session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019383467Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019392505Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"4c4ce03e-ad2b-4d6c-a016-5dcddd16f92c is now active with session: 4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019402474Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019411173Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"session_id":"4efc0ecc-2ed9-4edc-a1c7-50283a90b8c7"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019420302Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686} {"msg":"Activated sub_volume 4c4ce03e-ad2b-4d6c-a016-5dcddd16f92c","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019428271Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.01943644Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.019717438Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"task":"crutest"} 01/30 Write block 51 len 36864 data: 14 12 12 14 14 17 13 13 10 02/30 Read block 461 len 40960 03/30 Read block 236 len 4096 04/30 Read block 68 len 16384 05/30 Write block 266 len 40960 data: 9 12 13 15 15 12 12 13 13 15 06/30 Write block 72 len 36864 data: 8 9 10 10 10 8 11 9 11 07/30 Write block 198 len 4096 data: 12 08/30 Write block 529 len 28672 data: 8 11 11 12 14 13 13 09/30 Read block 227 len 20480 10/30 Read block 418 len 36864 11/30 Read block 27 len 24576 12/30 Read block 31 len 24576 13/30 Read block 239 len 12288 14/30 Write block 319 len 20480 data: 16 14 15 12 10 15/30 Write block 144 len 32768 data: 13 14 12 14 15 12 13 14 16/30 Write block 293 len 40960 data: 12 14 13 13 14 14 12 15 15 14 17/30 Write block 289 len 12288 data: 8 9 9 18/30 Write block 245 len 16384 data: 13 12 11 12 19/30 Read block 178 len 12288 20/30 Read block 500 len 40960 21/30 Read block 221 len 8192 22/30 Read block 251 len 32768 23/30 Write block 410 len 32768 data: 9 9 10 9 9 9 10 9 24/30 Read block 370 len 12288 25/30 Write block 565 len 28672 data: 10 10 11 12 7 9 12 26/30 Read block 207 len 36864 27/30 Read block 71 len 36864 28/30 Read block 355 len 4096 29/30 Write block 4 len 40960 data: 12 13 14 13 14 12 13 12 12 11 30/30 Read block 414 len 4096 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 10 Done Sent Done false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 1 27 Done 30 4 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:06.028598731Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:06.02877941Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1686,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 133 133 133 245 245 245 T F T <--- 1 020-039 134 134 134 246 246 246 F F F 2 040-059 134 134 134 246 246 246 T T T 3 060-079 134 134 134 246 246 246 T F T <--- 4 080-099 127 127 127 237 237 237 T F T <--- 5 100-119 130 130 130 241 241 241 F F F 6 120-139 130 130 130 241 241 241 F F F 7 140-159 128 128 128 239 239 239 T F T <--- 8 160-179 134 134 134 246 246 246 F F F 9 180-199 133 133 133 245 245 245 T F T <--- 10 200-219 123 123 123 228 228 228 F F F 11 220-239 134 134 134 246 246 246 F F F 12 240-259 134 134 134 246 246 246 T F T <--- 13 260-279 129 129 129 240 240 240 T F T <--- 14 280-299 131 131 131 243 243 243 T F T <--- 15 300-319 132 132 132 244 244 244 T F T <--- 16 320-339 134 134 134 246 246 246 T F T <--- 17 340-359 132 132 132 244 244 244 F F F 18 360-379 130 130 130 241 241 241 F F F 19 380-399 134 134 134 246 246 246 F F F 20 400-419 134 134 134 246 246 246 T F T <--- 21 420-439 131 131 131 243 243 243 F F F 22 440-459 133 133 133 245 245 245 F F F 23 460-479 129 129 129 240 240 240 F F F 24 480-499 134 134 134 246 246 246 F F F 25 500-519 134 134 134 246 246 246 F F F 26 520-539 130 130 130 242 242 242 T F T <--- 27 540-559 134 134 134 246 246 246 F F F 28 560-579 134 134 134 246 246 246 T F T <--- 29 580-599 133 133 133 245 245 245 F F F Max gen: 134, Max flush: 246 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 247 247 247 F F F 1 020-039 134 134 134 246 246 246 F F F 2 040-059 135 135 135 247 247 247 F F F 3 060-079 135 135 135 247 247 247 F F F 4 080-099 135 135 135 247 247 247 F F F 5 100-119 130 130 130 241 241 241 F F F 6 120-139 130 130 130 241 241 241 F F F 7 140-159 135 135 135 247 247 247 F F F 8 160-179 134 134 134 246 246 246 F F F 9 180-199 135 135 135 247 247 247 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 134 134 134 246 246 246 F F F 12 240-259 135 135 135 247 247 247 F F F 13 260-279 135 135 135 247 247 247 F F F 14 280-299 135 135 135 247 247 247 F F F 15 300-319 135 135 135 247 247 247 F F F 16 320-339 135 135 135 247 247 247 F F F 17 340-359 132 132 132 244 244 244 F F F 18 360-379 130 130 130 241 241 241 F F F 19 380-399 134 134 134 246 246 246 F F F 20 400-419 135 135 135 247 247 247 F F F 21 420-439 131 131 131 243 243 243 F F F 22 440-459 133 133 133 245 245 245 F F F 23 460-479 129 129 129 240 240 240 F F F 24 480-499 134 134 134 246 246 246 F F F 25 500-519 134 134 134 246 246 246 F F F 26 520-539 135 135 135 247 247 247 F F F 27 540-559 134 134 134 246 246 246 F F F 28 560-579 135 135 135 247 247 247 F F F 29 580-599 133 133 133 245 245 245 F F F Max gen: 135, Max flush: 247 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.286404802Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.294678566Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.294979451Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295004328Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295012967Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295024495Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694} {"msg":"Crucible e9687e58-5c2a-4cd0-b263-05fa67d85505 has session id: 6db90036-82fa-4943-ab8a-ba6a1510a773","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295359656Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"Upstairs opts: Upstairs UUID: e9687e58-5c2a-4cd0-b263-05fa67d85505, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295381384Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"Crucible stats registered with UUID: e9687e58-5c2a-4cd0-b263-05fa67d85505","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295391413Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:10.295400581Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295558763Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.29558789Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295599288Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295713635Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295728913Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694} {"msg":"e9687e58-5c2a-4cd0-b263-05fa67d85505 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295760989Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295772428Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"client":"0","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295782407Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"client":"1","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295792106Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"client":"2","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295801675Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295811694Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295821672Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"ds_connection connected from Ok(127.0.0.1:36055)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295896214Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"ds_connection connected from Ok(127.0.0.1:38376)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295921701Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"ds_connection connected from Ok(127.0.0.1:60456)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.295933069Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.296556197Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"client":"0","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.296576374Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"client":"1","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.296587583Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"client":"2","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.296928883Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.296982067Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"[0]R flush_numbers[0..12]: [247, 246, 247, 247, 247, 241, 241, 247, 246, 247, 228, 246]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.29704388Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"downstairs","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"[0]R generation[0..12]: [135, 134, 135, 135, 135, 130, 130, 135, 134, 135, 123, 134]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297058608Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"downstairs","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297069727Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"downstairs","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"[1]R flush_numbers[0..12]: [247, 246, 247, 247, 247, 241, 241, 247, 246, 247, 228, 246]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297080525Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"downstairs","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"[1]R generation[0..12]: [135, 134, 135, 135, 135, 130, 130, 135, 134, 135, 123, 134]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297091174Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"downstairs","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297102173Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"downstairs","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"[2]R flush_numbers[0..12]: [247, 246, 247, 247, 247, 241, 241, 247, 246, 247, 228, 246]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297112902Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"downstairs","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"[2]R generation[0..12]: [135, 134, 135, 135, 135, 130, 130, 135, 134, 135, 123, 134]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.29712354Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"downstairs","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297134379Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"downstairs","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297144028Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"downstairs","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"Generation requested: 138 >= found:136","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297153807Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"downstairs","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"Next flush: 248","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297163306Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"downstairs","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297172865Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"":"downstairs","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297182933Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297191842Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"client":"0","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297202981Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"client":"1","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297213Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"client":"2","":"downstairs","session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297223359Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297232608Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"e9687e58-5c2a-4cd0-b263-05fa67d85505 is now active with session: 6db90036-82fa-4943-ab8a-ba6a1510a773","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297242586Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297251405Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"session_id":"6db90036-82fa-4943-ab8a-ba6a1510a773"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297260604Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694} {"msg":"Activated sub_volume e9687e58-5c2a-4cd0-b263-05fa67d85505","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297268623Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297276822Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.297569118Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"task":"crutest"} 01/30 Write block 302 len 16384 data: 15 16 13 17 02/30 Write block 129 len 24576 data: 13 13 11 11 10 8 03/30 Write block 585 len 16384 data: 9 8 8 7 04/30 Write block 473 len 24576 data: 8 10 10 12 12 12 05/30 Write block 443 len 32768 data: 11 12 13 13 11 10 9 9 06/30 Read block 8 len 36864 07/30 Read block 288 len 28672 08/30 Write block 231 len 40960 data: 8 9 9 9 13 12 15 16 16 14 09/30 Flush 10/30 Flush 11/30 Write block 111 len 12288 data: 7 7 6 12/30 Read block 584 len 8192 13/30 Write block 118 len 24576 data: 14 14 13 13 13 14 14/30 Write block 161 len 16384 data: 10 7 9 10 15/30 Write block 115 len 12288 data: 7 10 11 16/30 Write block 451 len 8192 data: 6 5 17/30 Write block 52 len 12288 data: 13 13 15 18/30 Write block 55 len 28672 data: 15 18 14 14 11 10 10 19/30 Read block 441 len 8192 20/30 Read block 5 len 36864 21/30 Read block 325 len 12288 22/30 Write block 552 len 40960 data: 15 16 15 17 15 12 12 12 9 9 23/30 Flush 24/30 Write block 269 len 12288 data: 16 16 13 25/30 Write block 181 len 20480 data: 13 11 12 10 10 26/30 Read block 545 len 24576 27/30 Read block 16 len 40960 28/30 Write block 290 len 8192 data: 10 10 29/30 Write block 223 len 4096 data: 12 30/30 Write block 7 len 8192 data: 14 15 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Write 2 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 25 3 31 Done 27 5 27 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:10.304795025Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:10.304988462Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1694,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 135 135 135 247 247 247 T F T <--- 1 020-039 134 134 134 246 246 246 F F F 2 040-059 138 135 138 250 247 250 F F F <--- 3 060-079 138 135 138 250 247 250 F F F <--- 4 080-099 135 135 135 247 247 247 F F F 5 100-119 138 130 138 250 241 250 F F F <--- 6 120-139 138 130 138 250 241 250 F T F <--- 7 140-159 135 135 135 247 247 247 F F F 8 160-179 138 134 138 250 246 250 F F F <--- 9 180-199 135 135 135 247 247 247 T F T <--- 10 200-219 123 123 123 228 228 228 F F F 11 220-239 138 134 138 248 246 248 T F T <--- 12 240-259 138 135 138 248 247 248 F F F <--- 13 260-279 135 135 135 247 247 247 T F T <--- 14 280-299 135 135 135 247 247 247 T F T <--- 15 300-319 138 135 138 248 247 248 F T F <--- 16 320-339 135 135 135 247 247 247 F F F 17 340-359 132 132 132 244 244 244 F F F 18 360-379 130 130 130 241 241 241 F F F 19 380-399 134 134 134 246 246 246 F F F 20 400-419 135 135 135 247 247 247 F F F 21 420-439 131 131 131 243 243 243 F F F 22 440-459 138 133 138 250 245 250 F T F <--- 23 460-479 138 129 138 248 240 248 F T F <--- 24 480-499 134 134 134 246 246 246 F F F 25 500-519 134 134 134 246 246 246 F F F 26 520-539 135 135 135 247 247 247 F F F 27 540-559 138 134 138 250 246 250 F F F <--- 28 560-579 138 135 138 250 247 250 F F F <--- 29 580-599 138 133 138 248 245 248 F T F <--- Max gen: 138, Max flush: 250 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 139 139 139 251 251 251 F F F 1 020-039 134 134 134 246 246 246 F F F 2 040-059 138 138 138 250 250 250 F F F 3 060-079 138 138 138 250 250 250 F F F 4 080-099 135 135 135 247 247 247 F F F 5 100-119 138 138 138 250 250 250 F F F 6 120-139 138 138 138 250 250 250 F F F 7 140-159 135 135 135 247 247 247 F F F 8 160-179 138 138 138 250 250 250 F F F 9 180-199 139 139 139 251 251 251 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 139 139 139 251 251 251 F F F 12 240-259 138 138 138 248 248 248 F F F 13 260-279 139 139 139 251 251 251 F F F 14 280-299 139 139 139 251 251 251 F F F 15 300-319 138 138 138 248 248 248 F F F 16 320-339 135 135 135 247 247 247 F F F 17 340-359 132 132 132 244 244 244 F F F 18 360-379 130 130 130 241 241 241 F F F 19 380-399 134 134 134 246 246 246 F F F 20 400-419 135 135 135 247 247 247 F F F 21 420-439 131 131 131 243 243 243 F F F 22 440-459 138 138 138 250 250 250 F F F 23 460-479 138 138 138 248 248 248 F F F 24 480-499 134 134 134 246 246 246 F F F 25 500-519 134 134 134 246 246 246 F F F 26 520-539 135 135 135 247 247 247 F F F 27 540-559 138 138 138 250 250 250 F F F 28 560-579 138 138 138 250 250 250 F F F 29 580-599 138 138 138 248 248 248 F F F Max gen: 139, Max flush: 251 Begin loop 70 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.56787835Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.575913142Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.576211587Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.576247413Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.576256642Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.57626836Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702} {"msg":"Crucible b00053d4-fa29-4ea0-8722-0f13cb9d4690 has session id: 8d58ac55-2f75-4549-bf24-3da5bfb15779","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.576551327Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"Upstairs opts: Upstairs UUID: b00053d4-fa29-4ea0-8722-0f13cb9d4690, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.576626938Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"Crucible stats registered with UUID: b00053d4-fa29-4ea0-8722-0f13cb9d4690","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.576638317Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:14.576647476Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.576802688Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.576833004Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.576843843Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.576961509Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.576976568Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702} {"msg":"b00053d4-fa29-4ea0-8722-0f13cb9d4690 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.577000245Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.577009814Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"client":"0","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.577019443Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"client":"1","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.577029041Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"client":"2","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.57703842Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.577048589Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.577060628Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"ds_connection connected from Ok(127.0.0.1:44349)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.577115361Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"ds_connection connected from Ok(127.0.0.1:65028)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.57713136Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"ds_connection connected from Ok(127.0.0.1:56358)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.577201241Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.577795742Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"client":"1","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.57781458Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"client":"2","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.577887171Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"client":"0","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578113395Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578144171Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"[0]R flush_numbers[0..12]: [251, 246, 250, 250, 247, 250, 250, 247, 250, 251, 228, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578228811Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"downstairs","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"[0]R generation[0..12]: [139, 134, 138, 138, 135, 138, 138, 135, 138, 139, 123, 139]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.57824004Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"downstairs","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578251139Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"downstairs","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"[1]R flush_numbers[0..12]: [251, 246, 250, 250, 247, 250, 250, 247, 250, 251, 228, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578261768Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"downstairs","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"[1]R generation[0..12]: [139, 134, 138, 138, 135, 138, 138, 135, 138, 139, 123, 139]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578272386Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"downstairs","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578286095Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"downstairs","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"[2]R flush_numbers[0..12]: [251, 246, 250, 250, 247, 250, 250, 247, 250, 251, 228, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578296653Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"downstairs","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"[2]R generation[0..12]: [139, 134, 138, 138, 135, 138, 138, 135, 138, 139, 123, 139]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578307102Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"downstairs","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578318001Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"downstairs","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.57832751Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"downstairs","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578337319Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"downstairs","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578346798Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"downstairs","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578356207Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"":"downstairs","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578365895Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578374674Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"client":"0","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578384233Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"client":"1","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578393912Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"client":"2","":"downstairs","session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578403691Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.57841289Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"b00053d4-fa29-4ea0-8722-0f13cb9d4690 is now active with session: 8d58ac55-2f75-4549-bf24-3da5bfb15779","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578422779Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578432768Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"session_id":"8d58ac55-2f75-4549-bf24-3da5bfb15779"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578441747Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702} {"msg":"Activated sub_volume b00053d4-fa29-4ea0-8722-0f13cb9d4690","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578449716Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578457775Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.578767099Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"task":"crutest"} 01/30 Write block 115 len 32768 data: 8 11 12 15 15 14 14 14 02/30 Read block 553 len 24576 03/30 Write block 367 len 28672 data: 5 6 6 5 7 7 6 04/30 Write block 558 len 32768 data: 13 13 10 10 10 11 13 11 05/30 Read block 114 len 36864 06/30 Read block 507 len 20480 07/30 Read block 178 len 36864 08/30 Flush 09/30 Write block 17 len 8192 data: 9 8 10/30 Write block 61 len 36864 data: 11 9 9 10 10 9 9 9 9 11/30 Write block 461 len 20480 data: 6 7 7 7 6 12/30 Read block 426 len 4096 13/30 Write block 151 len 40960 data: 15 15 15 13 13 12 12 11 15 13 14/30 Write block 498 len 28672 data: 10 12 12 10 8 10 10 15/30 Read block 556 len 16384 16/30 Read block 97 len 8192 17/30 Write block 92 len 32768 data: 16 17 17 15 12 7 8 7 18/30 Write block 183 len 12288 data: 13 11 11 19/30 Write block 545 len 16384 data: 14 16 17 17 20/30 Write block 36 len 12288 data: 10 9 11 21/30 Flush 22/30 Write block 300 len 36864 data: 16 16 16 17 14 18 17 17 14 23/30 Write block 351 len 36864 data: 15 16 16 16 12 9 7 8 8 24/30 Read block 460 len 4096 25/30 Read block 247 len 8192 26/30 Write block 287 len 12288 data: 8 7 9 27/30 Read block 476 len 4096 28/30 Read block 531 len 40960 29/30 Read block 98 len 4096 30/30 Read block 120 len 20480 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 6 Done Done Sent false 1002 Acked Write 7 Done Done Sent false 1003 Acked Write 8 Done Done Sent false 1004 Acked Read 9 Done Done Sent false 1005 Acked Read 5 Done Done Sent false 1006 Acked Read 9 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Write 9 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:14.587128743Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:14.587297743Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1702,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 139 253 253 251 F F F <--- 1 020-039 140 140 134 253 253 246 F F F <--- 2 040-059 138 138 138 250 250 250 F F F 3 060-079 140 140 138 253 253 250 F F F <--- 4 080-099 140 140 135 253 253 247 F F F <--- 5 100-119 140 140 138 252 252 250 F F T <--- 6 120-139 140 140 138 252 252 250 F F T <--- 7 140-159 140 140 135 253 253 247 F F F <--- 8 160-179 140 140 138 253 253 250 F F F <--- 9 180-199 140 140 139 253 253 251 F F F <--- 10 200-219 123 123 123 228 228 228 F F F 11 220-239 139 139 139 251 251 251 F F F 12 240-259 138 138 138 248 248 248 F F F 13 260-279 139 139 139 251 251 251 F F F 14 280-299 139 139 139 251 251 251 T T F <--- 15 300-319 138 138 138 248 248 248 T T F <--- 16 320-339 135 135 135 247 247 247 F F F 17 340-359 132 132 132 244 244 244 T T F <--- 18 360-379 140 140 130 252 252 241 F F F <--- 19 380-399 134 134 134 246 246 246 F F F 20 400-419 135 135 135 247 247 247 F F F 21 420-439 131 131 131 243 243 243 F F F 22 440-459 138 138 138 250 250 250 F F F 23 460-479 140 140 138 253 253 248 F F F <--- 24 480-499 140 140 134 253 253 246 F F F <--- 25 500-519 140 140 134 253 253 246 F F F <--- 26 520-539 135 135 135 247 247 247 F F F 27 540-559 140 140 138 253 253 250 F F F <--- 28 560-579 140 140 138 252 252 250 F F F <--- 29 580-599 138 138 138 248 248 248 F F F Max gen: 140, Max flush: 253 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 253 253 253 F F F 1 020-039 140 140 140 253 253 253 F F F 2 040-059 138 138 138 250 250 250 F F F 3 060-079 140 140 140 253 253 253 F F F 4 080-099 140 140 140 253 253 253 F F F 5 100-119 140 140 140 252 252 252 F F F 6 120-139 140 140 140 252 252 252 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 140 140 140 253 253 253 F F F 9 180-199 140 140 140 253 253 253 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 139 139 139 251 251 251 F F F 12 240-259 138 138 138 248 248 248 F F F 13 260-279 139 139 139 251 251 251 F F F 14 280-299 141 141 141 254 254 254 F F F 15 300-319 141 141 141 254 254 254 F F F 16 320-339 135 135 135 247 247 247 F F F 17 340-359 141 141 141 254 254 254 F F F 18 360-379 140 140 140 252 252 252 F F F 19 380-399 134 134 134 246 246 246 F F F 20 400-419 135 135 135 247 247 247 F F F 21 420-439 131 131 131 243 243 243 F F F 22 440-459 138 138 138 250 250 250 F F F 23 460-479 140 140 140 253 253 253 F F F 24 480-499 140 140 140 253 253 253 F F F 25 500-519 140 140 140 253 253 253 F F F 26 520-539 135 135 135 247 247 247 F F F 27 540-559 140 140 140 253 253 253 F F F 28 560-579 140 140 140 252 252 252 F F F 29 580-599 138 138 138 248 248 248 F F F Max gen: 141, Max flush: 254 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.851689004Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.859718557Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.859998654Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860040549Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860049668Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860061397Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710} {"msg":"Crucible bec1b354-9406-4163-a7fc-f5d18acd4ded has session id: cca554fa-39a5-4e4d-bc49-c12e9b236560","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860365142Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"Upstairs opts: Upstairs UUID: bec1b354-9406-4163-a7fc-f5d18acd4ded, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860417545Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"Crucible stats registered with UUID: bec1b354-9406-4163-a7fc-f5d18acd4ded","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860428204Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:18.860439373Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860569298Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860583936Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860593915Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860729719Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860748867Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710} {"msg":"bec1b354-9406-4163-a7fc-f5d18acd4ded active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860775194Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860784503Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"client":"0","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860794301Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"client":"1","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.86080384Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"client":"2","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860813529Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860823458Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860833177Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"ds_connection connected from Ok(127.0.0.1:39286)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860906288Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"ds_connection connected from Ok(127.0.0.1:38118)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860931495Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"ds_connection connected from Ok(127.0.0.1:33315)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.860942694Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.861556752Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"client":"0","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.861583119Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"client":"1","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.861603807Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"client":"2","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.861869646Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.861895033Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"[0]R flush_numbers[0..12]: [253, 253, 250, 253, 253, 252, 252, 253, 253, 253, 228, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.861980983Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"downstairs","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"[0]R generation[0..12]: [140, 140, 138, 140, 140, 140, 140, 140, 140, 140, 123, 139]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.861992212Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"downstairs","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.86200316Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"downstairs","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"[1]R flush_numbers[0..12]: [253, 253, 250, 253, 253, 252, 252, 253, 253, 253, 228, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862014179Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"downstairs","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"[1]R generation[0..12]: [140, 140, 138, 140, 140, 140, 140, 140, 140, 140, 123, 139]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862024838Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"downstairs","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862035746Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"downstairs","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"[2]R flush_numbers[0..12]: [253, 253, 250, 253, 253, 252, 252, 253, 253, 253, 228, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862046165Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"downstairs","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"[2]R generation[0..12]: [140, 140, 138, 140, 140, 140, 140, 140, 140, 140, 123, 139]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862056594Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"downstairs","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862067493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"downstairs","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862079481Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"downstairs","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.86208929Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"downstairs","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862098519Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"downstairs","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862107868Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"":"downstairs","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862117537Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862126296Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"client":"0","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862136015Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"client":"1","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862145484Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"client":"2","":"downstairs","session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862155273Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862164311Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"bec1b354-9406-4163-a7fc-f5d18acd4ded is now active with session: cca554fa-39a5-4e4d-bc49-c12e9b236560","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.86217419Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862182979Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"session_id":"cca554fa-39a5-4e4d-bc49-c12e9b236560"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862191988Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710} {"msg":"Activated sub_volume bec1b354-9406-4163-a7fc-f5d18acd4ded","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862200007Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862208076Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.862532369Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"task":"crutest"} 01/30 Write block 404 len 36864 data: 9 6 6 8 8 9 10 10 11 02/30 Write block 49 len 12288 data: 10 9 15 03/30 Read block 424 len 28672 04/30 Write block 509 len 8192 data: 9 8 05/30 Write block 307 len 4096 data: 18 06/30 Read block 417 len 40960 07/30 Read block 88 len 36864 08/30 Write block 110 len 4096 data: 7 09/30 Write block 547 len 40960 data: 18 18 16 17 15 16 17 16 18 16 10/30 Read block 87 len 28672 11/30 Read block 376 len 24576 12/30 Write block 240 len 20480 data: 15 17 18 15 16 13/30 Read block 58 len 8192 14/30 Write block 124 len 36864 data: 14 12 13 12 12 14 14 12 12 15/30 Write block 476 len 32768 data: 13 13 13 12 8 11 9 8 16/30 Read block 177 len 16384 17/30 Write block 366 len 12288 data: 6 6 7 18/30 Flush 19/30 Write block 437 len 24576 data: 7 8 10 9 13 14 20/30 Read block 572 len 8192 21/30 Read block 455 len 4096 22/30 Write block 60 len 4096 data: 11 23/30 Read block 146 len 24576 24/30 Read block 303 len 24576 25/30 Read block 535 len 16384 26/30 Write block 523 len 24576 data: 11 10 9 5 5 6 27/30 Write block 549 len 12288 data: 17 18 16 28/30 Read block 518 len 16384 29/30 Write block 247 len 16384 data: 12 13 10 12 30/30 Write block 261 len 16384 data: 6 7 8 8 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Sent Done false 1001 Acked Write 3 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Write 2 Done Sent Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Write 1 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Read 4 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:18.87082904Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:18.871045475Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1710,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 253 253 253 F F F 1 020-039 140 140 140 253 253 253 F F F 2 040-059 142 138 142 255 250 255 F F F <--- 3 060-079 140 140 140 253 253 253 T F T <--- 4 080-099 140 140 140 253 253 253 F F F 5 100-119 142 140 142 255 252 255 F F F <--- 6 120-139 142 140 142 255 252 255 F F F <--- 7 140-159 140 140 140 253 253 253 F F F 8 160-179 140 140 140 253 253 253 F F F 9 180-199 140 140 140 253 253 253 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 139 139 139 251 251 251 F F F 12 240-259 142 138 142 255 248 255 T F T <--- 13 260-279 139 139 139 251 251 251 T F T <--- 14 280-299 141 141 141 254 254 254 F F F 15 300-319 142 141 142 255 254 255 F F F <--- 16 320-339 135 135 135 247 247 247 F F F 17 340-359 141 141 141 254 254 254 F F F 18 360-379 142 140 142 255 252 255 F F F <--- 19 380-399 134 134 134 246 246 246 F F F 20 400-419 142 135 142 255 247 255 F F F <--- 21 420-439 131 131 131 243 243 243 T F T <--- 22 440-459 138 138 138 250 250 250 T F T <--- 23 460-479 142 140 142 255 253 255 F F F <--- 24 480-499 142 140 142 255 253 255 F F F <--- 25 500-519 142 140 142 255 253 255 F F F <--- 26 520-539 135 135 135 247 247 247 T F T <--- 27 540-559 142 140 142 255 253 255 T F T <--- 28 560-579 140 140 140 252 252 252 F F F 29 580-599 138 138 138 248 248 248 F F F Max gen: 142, Max flush: 255 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 140 140 253 253 253 F F F 1 020-039 140 140 140 253 253 253 F F F 2 040-059 142 142 142 255 255 255 F F F 3 060-079 143 143 143 256 256 256 F F F 4 080-099 140 140 140 253 253 253 F F F 5 100-119 142 142 142 255 255 255 F F F 6 120-139 142 142 142 255 255 255 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 140 140 140 253 253 253 F F F 9 180-199 140 140 140 253 253 253 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 139 139 139 251 251 251 F F F 12 240-259 143 143 143 256 256 256 F F F 13 260-279 143 143 143 256 256 256 F F F 14 280-299 141 141 141 254 254 254 F F F 15 300-319 142 142 142 255 255 255 F F F 16 320-339 135 135 135 247 247 247 F F F 17 340-359 141 141 141 254 254 254 F F F 18 360-379 142 142 142 255 255 255 F F F 19 380-399 134 134 134 246 246 246 F F F 20 400-419 142 142 142 255 255 255 F F F 21 420-439 143 143 143 256 256 256 F F F 22 440-459 143 143 143 256 256 256 F F F 23 460-479 142 142 142 255 255 255 F F F 24 480-499 142 142 142 255 255 255 F F F 25 500-519 142 142 142 255 255 255 F F F 26 520-539 143 143 143 256 256 256 F F F 27 540-559 143 143 143 256 256 256 F F F 28 560-579 140 140 140 252 252 252 F F F 29 580-599 138 138 138 248 248 248 F F F Max gen: 143, Max flush: 256 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.128199332Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:23.135481672Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.135541845Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.142631307Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.142915284Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.14294993Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.142959269Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.142971017Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718} {"msg":"Crucible 60c9f1e7-a5f8-4e4c-9a97-834d072a7abc has session id: 4fdfd6e9-2882-4176-bc9a-c00a03616edf","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143311248Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"Upstairs opts: Upstairs UUID: 60c9f1e7-a5f8-4e4c-9a97-834d072a7abc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143333575Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"Crucible stats registered with UUID: 60c9f1e7-a5f8-4e4c-9a97-834d072a7abc","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143343414Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:23.143352423Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143520573Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.14355017Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143563578Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143691533Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143707441Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718} {"msg":"60c9f1e7-a5f8-4e4c-9a97-834d072a7abc active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143730939Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143740318Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"client":"0","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143750106Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"client":"1","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143759845Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"client":"2","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143769234Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143779243Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143789202Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"ds_connection connected from Ok(127.0.0.1:64912)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143842936Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"ds_connection connected from Ok(127.0.0.1:50057)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143854104Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"ds_connection connected from Ok(127.0.0.1:46085)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.143868973Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.14448984Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"client":"1","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.144557872Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"client":"0","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.144596298Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"client":"2","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.144881404Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.144891253Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"[0]R flush_numbers[0..12]: [253, 253, 255, 256, 253, 255, 255, 253, 253, 253, 228, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.144910221Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"downstairs","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"[0]R generation[0..12]: [140, 140, 142, 143, 140, 142, 142, 140, 140, 140, 123, 139]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.14492077Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"downstairs","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.144931638Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"downstairs","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"[1]R flush_numbers[0..12]: [253, 253, 255, 256, 253, 255, 255, 253, 253, 253, 228, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.144942307Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"downstairs","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"[1]R generation[0..12]: [140, 140, 142, 143, 140, 142, 142, 140, 140, 140, 123, 139]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.144952816Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"downstairs","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.144963585Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"downstairs","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"[2]R flush_numbers[0..12]: [253, 253, 255, 256, 253, 255, 255, 253, 253, 253, 228, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.144974224Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"downstairs","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"[2]R generation[0..12]: [140, 140, 142, 143, 140, 142, 142, 140, 140, 140, 123, 139]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.144984712Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"downstairs","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.144995461Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"downstairs","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.14500482Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"downstairs","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145014779Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"downstairs","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145024268Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"downstairs","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145035126Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"":"downstairs","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145044925Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145053694Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"client":"0","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145063343Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"client":"1","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145072952Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"client":"2","":"downstairs","session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145082701Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.14509183Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"60c9f1e7-a5f8-4e4c-9a97-834d072a7abc is now active with session: 4fdfd6e9-2882-4176-bc9a-c00a03616edf","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145101759Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145110518Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"session_id":"4fdfd6e9-2882-4176-bc9a-c00a03616edf"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145119747Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718} {"msg":"Activated sub_volume 60c9f1e7-a5f8-4e4c-9a97-834d072a7abc","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145127706Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145135845Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.145470256Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"task":"crutest"} 01/30 Write block 1 len 28672 data: 6 9 9 13 14 15 15 02/30 Read block 169 len 4096 03/30 Write block 111 len 4096 data: 8 04/30 Write block 413 len 8192 data: 10 10 05/30 Read block 447 len 28672 06/30 Write block 37 len 32768 data: 10 12 10 12 11 10 9 10 07/30 Write block 363 len 24576 data: 7 8 7 7 7 8 08/30 Write block 83 len 4096 data: 12 09/30 Read block 500 len 28672 10/30 Read block 453 len 24576 11/30 Write block 43 len 36864 data: 10 11 12 13 12 9 11 10 16 12/30 Write block 573 len 20480 data: 12 13 13 13 14 13/30 Read block 474 len 12288 14/30 Read block 233 len 28672 15/30 Write block 53 len 40960 data: 14 16 16 19 15 15 12 12 12 10 16/30 Read block 30 len 8192 17/30 Read block 147 len 28672 18/30 Flush 19/30 Write block 335 len 12288 data: 13 12 13 20/30 Read block 190 len 36864 21/30 Read block 417 len 12288 22/30 Write block 516 len 36864 data: 12 13 14 13 11 11 11 12 11 23/30 Write block 376 len 32768 data: 10 10 9 9 10 9 10 12 24/30 Read block 125 len 12288 25/30 Read block 359 len 28672 26/30 Read block 136 len 36864 27/30 Read block 178 len 32768 28/30 Write block 243 len 36864 data: 16 17 14 13 13 14 11 13 13 29/30 Read block 486 len 4096 30/30 Read block 242 len 20480 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Sent Done Done false 1001 Acked Read 1 Sent Done Done false 1002 Acked Write 1 Sent Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Read 7 Sent Done Done false 1009 Acked Read 6 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:23.155022081Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:23.155162104Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1718,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 140 144 144 253 257 257 F F F <--- 1 020-039 140 144 144 253 257 257 F F F <--- 2 040-059 142 144 144 255 257 257 F F F <--- 3 060-079 143 144 144 256 257 257 F F F <--- 4 080-099 140 144 144 253 257 257 F F F <--- 5 100-119 142 144 144 255 257 257 F F F <--- 6 120-139 142 142 142 255 255 255 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 140 140 140 253 253 253 F F F 9 180-199 140 140 140 253 253 253 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 139 139 139 251 251 251 F F F 12 240-259 143 143 143 256 256 256 F T T <--- 13 260-279 143 143 143 256 256 256 F F F 14 280-299 141 141 141 254 254 254 F F F 15 300-319 142 142 142 255 255 255 F F F 16 320-339 135 135 135 247 247 247 F T T <--- 17 340-359 141 141 141 254 254 254 F F F 18 360-379 142 144 144 255 257 257 F T T <--- 19 380-399 134 134 134 246 246 246 F T T <--- 20 400-419 142 144 144 255 257 257 F F F <--- 21 420-439 143 143 143 256 256 256 F F F 22 440-459 143 143 143 256 256 256 F F F 23 460-479 142 142 142 255 255 255 F F F 24 480-499 142 142 142 255 255 255 F F F 25 500-519 142 142 142 255 255 255 F T T <--- 26 520-539 143 143 143 256 256 256 F T T <--- 27 540-559 143 143 143 256 256 256 F F F 28 560-579 140 144 144 252 257 257 F F F <--- 29 580-599 138 138 138 248 248 248 F F F Max gen: 144, Max flush: 257 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 257 257 257 F F F 1 020-039 144 144 144 257 257 257 F F F 2 040-059 144 144 144 257 257 257 F F F 3 060-079 144 144 144 257 257 257 F F F 4 080-099 144 144 144 257 257 257 F F F 5 100-119 144 144 144 257 257 257 F F F 6 120-139 142 142 142 255 255 255 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 140 140 140 253 253 253 F F F 9 180-199 140 140 140 253 253 253 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 139 139 139 251 251 251 F F F 12 240-259 145 145 145 258 258 258 F F F 13 260-279 143 143 143 256 256 256 F F F 14 280-299 141 141 141 254 254 254 F F F 15 300-319 142 142 142 255 255 255 F F F 16 320-339 145 145 145 258 258 258 F F F 17 340-359 141 141 141 254 254 254 F F F 18 360-379 145 145 145 258 258 258 F F F 19 380-399 145 145 145 258 258 258 F F F 20 400-419 144 144 144 257 257 257 F F F 21 420-439 143 143 143 256 256 256 F F F 22 440-459 143 143 143 256 256 256 F F F 23 460-479 142 142 142 255 255 255 F F F 24 480-499 142 142 142 255 255 255 F F F 25 500-519 145 145 145 258 258 258 F F F 26 520-539 145 145 145 258 258 258 F F F 27 540-559 143 143 143 256 256 256 F F F 28 560-579 144 144 144 257 257 257 F F F 29 580-599 138 138 138 248 248 248 F F F Max gen: 145, Max flush: 258 Begin loop 73 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.41695115Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.425166731Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.42543568Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.425460407Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.425472235Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.425483734Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726} {"msg":"Crucible efc58833-3e96-4efc-8692-f8921a03c5ad has session id: c91c0a44-7f94-4ed7-a002-79893e20f53d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.425808446Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"Upstairs opts: Upstairs UUID: efc58833-3e96-4efc-8692-f8921a03c5ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.425828194Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"Crucible stats registered with UUID: efc58833-3e96-4efc-8692-f8921a03c5ad","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.425838533Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:27.425847632Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.425975967Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.425990655Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.426000774Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.426149496Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.426165045Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726} {"msg":"efc58833-3e96-4efc-8692-f8921a03c5ad active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.426182602Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.426191371Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"client":"0","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.42620108Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"client":"1","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.426212429Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"client":"2","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.426221878Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.426231727Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.426241516Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43441)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.42628933Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48943)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.426345783Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36022)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.426425244Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.426943014Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"client":"2","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.42697603Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"client":"1","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.426988018Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"client":"0","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.42723621Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427264946Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"[0]R flush_numbers[0..12]: [257, 257, 257, 257, 257, 257, 255, 253, 253, 253, 228, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427305121Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"downstairs","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"[0]R generation[0..12]: [144, 144, 144, 144, 144, 144, 142, 140, 140, 140, 123, 139]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.42731639Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"downstairs","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427327359Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"downstairs","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"[1]R flush_numbers[0..12]: [257, 257, 257, 257, 257, 257, 255, 253, 253, 253, 228, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427340717Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"downstairs","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"[1]R generation[0..12]: [144, 144, 144, 144, 144, 144, 142, 140, 140, 140, 123, 139]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427351366Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"downstairs","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427362305Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"downstairs","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"[2]R flush_numbers[0..12]: [257, 257, 257, 257, 257, 257, 255, 253, 253, 253, 228, 251]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427372914Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"downstairs","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"[2]R generation[0..12]: [144, 144, 144, 144, 144, 144, 142, 140, 140, 140, 123, 139]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427383342Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"downstairs","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427394191Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"downstairs","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.42740354Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"downstairs","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427413349Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"downstairs","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427422688Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"downstairs","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427432017Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"":"downstairs","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427441666Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427450464Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"client":"0","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427460203Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"client":"1","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427469872Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"client":"2","":"downstairs","session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427479721Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.42749036Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"efc58833-3e96-4efc-8692-f8921a03c5ad is now active with session: c91c0a44-7f94-4ed7-a002-79893e20f53d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427500329Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427509198Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"session_id":"c91c0a44-7f94-4ed7-a002-79893e20f53d"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427518167Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726} {"msg":"Activated sub_volume efc58833-3e96-4efc-8692-f8921a03c5ad","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427526206Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427539974Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.427784386Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"task":"crutest"} 01/30 Write block 61 len 24576 data: 13 11 10 11 11 10 02/30 Write block 24 len 20480 data: 13 12 16 14 17 03/30 Write block 90 len 32768 data: 16 15 17 18 18 16 13 8 04/30 Read block 337 len 20480 05/30 Write block 354 len 12288 data: 17 13 10 06/30 Read block 230 len 24576 07/30 Read block 1 len 24576 08/30 Write block 308 len 32768 data: 15 15 15 9 9 9 7 10 09/30 Read block 564 len 16384 10/30 Write block 165 len 36864 data: 11 11 9 11 11 8 10 10 10 11/30 Write block 436 len 24576 data: 11 8 9 11 10 14 12/30 Write block 175 len 32768 data: 6 8 8 14 14 14 14 12 13/30 Write block 134 len 24576 data: 9 10 8 7 9 10 14/30 Write block 582 len 36864 data: 12 10 8 10 9 9 8 11 10 15/30 Write block 389 len 28672 data: 9 8 10 9 10 12 12 16/30 Read block 297 len 20480 17/30 Write block 238 len 20480 data: 17 17 16 18 19 18/30 Flush 19/30 Write block 432 len 40960 data: 13 13 14 12 12 9 10 12 11 15 20/30 Read block 75 len 4096 21/30 Read block 33 len 28672 22/30 Read block 470 len 36864 23/30 Read block 29 len 28672 24/30 Read block 151 len 20480 25/30 Read block 591 len 8192 26/30 Write block 72 len 16384 data: 9 10 11 11 27/30 Write block 394 len 4096 data: 13 28/30 Write block 81 len 24576 data: 8 10 13 12 12 14 29/30 Read block 425 len 12288 30/30 Read block 438 len 36864 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Sent Done false 1001 Acked Write 5 Done Sent Done false 1002 Acked Write 8 Done Sent Done false 1003 Acked Read 5 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 6 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 0 31 Done 29 0 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:27.436334128Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:27.436513967Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1726,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 257 257 257 F F F 1 020-039 146 144 146 259 257 259 F F F <--- 2 040-059 144 144 144 257 257 257 F F F 3 060-079 146 144 146 259 257 259 T F T <--- 4 080-099 146 144 146 259 257 259 T F T <--- 5 100-119 144 144 144 257 257 257 F F F 6 120-139 146 142 146 259 255 259 F F F <--- 7 140-159 140 140 140 253 253 253 F F F 8 160-179 146 140 146 259 253 259 F F F <--- 9 180-199 146 140 146 259 253 259 F F F <--- 10 200-219 123 123 123 228 228 228 F F F 11 220-239 146 139 146 259 251 259 F F F <--- 12 240-259 146 145 146 259 258 259 F F F <--- 13 260-279 143 143 143 256 256 256 F F F 14 280-299 141 141 141 254 254 254 F F F 15 300-319 146 142 146 259 255 259 F F F <--- 16 320-339 145 145 145 258 258 258 F F F 17 340-359 146 141 146 259 254 259 F F F <--- 18 360-379 145 145 145 258 258 258 F F F 19 380-399 146 145 146 259 258 259 T F T <--- 20 400-419 144 144 144 257 257 257 F F F 21 420-439 146 143 146 259 256 259 T F T <--- 22 440-459 146 143 146 259 256 259 T F T <--- 23 460-479 142 142 142 255 255 255 F F F 24 480-499 142 142 142 255 255 255 F F F 25 500-519 145 145 145 258 258 258 F F F 26 520-539 145 145 145 258 258 258 F F F 27 540-559 143 143 143 256 256 256 F F F 28 560-579 144 144 144 257 257 257 F F F 29 580-599 146 138 146 259 248 259 F F F <--- Max gen: 146, Max flush: 259 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 257 257 257 F F F 1 020-039 146 146 146 259 259 259 F F F 2 040-059 144 144 144 257 257 257 F F F 3 060-079 147 147 147 260 260 260 F F F 4 080-099 147 147 147 260 260 260 F F F 5 100-119 144 144 144 257 257 257 F F F 6 120-139 146 146 146 259 259 259 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 146 146 146 259 259 259 F F F 9 180-199 146 146 146 259 259 259 F F F 10 200-219 123 123 123 228 228 228 F F F 11 220-239 146 146 146 259 259 259 F F F 12 240-259 146 146 146 259 259 259 F F F 13 260-279 143 143 143 256 256 256 F F F 14 280-299 141 141 141 254 254 254 F F F 15 300-319 146 146 146 259 259 259 F F F 16 320-339 145 145 145 258 258 258 F F F 17 340-359 146 146 146 259 259 259 F F F 18 360-379 145 145 145 258 258 258 F F F 19 380-399 147 147 147 260 260 260 F F F 20 400-419 144 144 144 257 257 257 F F F 21 420-439 147 147 147 260 260 260 F F F 22 440-459 147 147 147 260 260 260 F F F 23 460-479 142 142 142 255 255 255 F F F 24 480-499 142 142 142 255 255 255 F F F 25 500-519 145 145 145 258 258 258 F F F 26 520-539 145 145 145 258 258 258 F F F 27 540-559 143 143 143 256 256 256 F F F 28 560-579 144 144 144 257 257 257 F F F 29 580-599 146 146 146 259 259 259 F F F Max gen: 147, Max flush: 260 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.686662841Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:31.693707959Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.693763012Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.700594725Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.700845536Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.700869773Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.700878262Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.70088996Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734} {"msg":"Crucible 73bf1ced-60f9-4f07-9b5c-afac60cfc871 has session id: ae30304b-2137-4e58-8eba-9f204f316e08","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701255028Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"Upstairs opts: Upstairs UUID: 73bf1ced-60f9-4f07-9b5c-afac60cfc871, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701286954Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"Crucible stats registered with UUID: 73bf1ced-60f9-4f07-9b5c-afac60cfc871","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701297873Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:31.701307202Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701440726Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701456164Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701466263Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701599178Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701615986Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734} {"msg":"73bf1ced-60f9-4f07-9b5c-afac60cfc871 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701628164Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701636793Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"client":"0","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701646502Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"client":"1","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701656261Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"client":"2","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.70166582Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701676199Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701688557Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"ds_connection connected from Ok(127.0.0.1:51788)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701732312Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"ds_connection connected from Ok(127.0.0.1:37990)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701757979Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"ds_connection connected from Ok(127.0.0.1:53129)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.701771867Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702418312Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"client":"1","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702465406Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"client":"2","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702525279Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"client":"0","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702759332Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702783769Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"[0]R flush_numbers[0..12]: [257, 259, 257, 260, 260, 257, 259, 253, 259, 259, 228, 259]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702848572Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"downstairs","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"[0]R generation[0..12]: [144, 146, 144, 147, 147, 144, 146, 140, 146, 146, 123, 146]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.70286047Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"downstairs","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702872059Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"downstairs","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"[1]R flush_numbers[0..12]: [257, 259, 257, 260, 260, 257, 259, 253, 259, 259, 228, 259]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702882748Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"downstairs","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"[1]R generation[0..12]: [144, 146, 144, 147, 147, 144, 146, 140, 146, 146, 123, 146]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702893606Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"downstairs","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702907195Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"downstairs","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"[2]R flush_numbers[0..12]: [257, 259, 257, 260, 260, 257, 259, 253, 259, 259, 228, 259]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702917884Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"downstairs","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"[2]R generation[0..12]: [144, 146, 144, 147, 147, 144, 146, 140, 146, 146, 123, 146]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702928482Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"downstairs","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702939291Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"downstairs","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.70294891Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"downstairs","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702958859Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"downstairs","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702968268Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"downstairs","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702977577Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"":"downstairs","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702987246Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.702995984Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"client":"0","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.703005703Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"client":"1","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.703015312Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"client":"2","":"downstairs","session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.703025101Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.70303418Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"73bf1ced-60f9-4f07-9b5c-afac60cfc871 is now active with session: ae30304b-2137-4e58-8eba-9f204f316e08","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.703044169Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.703052848Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"session_id":"ae30304b-2137-4e58-8eba-9f204f316e08"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.703063297Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Activated sub_volume 73bf1ced-60f9-4f07-9b5c-afac60cfc871","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.703076565Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.703084854Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.703328186Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"task":"crutest"} 01/30 Write block 439 len 24576 data: 13 12 16 15 12 13 02/30 Read block 452 len 8192 03/30 Write block 545 len 28672 data: 15 17 19 19 18 19 17 04/30 Flush 05/30 Write block 250 len 16384 data: 14 14 13 12 06/30 Write block 120 len 32768 data: 15 15 15 15 15 13 14 13 07/30 Flush 08/30 Write block 372 len 20480 data: 8 7 7 9 11 09/30 Read block 276 len 36864 10/30 Read block 184 len 12288 11/30 Write block 202 len 32768 data: 9 10 10 9 8 7 10 13 12/30 Write block 413 len 28672 data: 11 11 10 11 10 9 10 13/30 Read block 235 len 20480 14/30 Read block 225 len 28672 15/30 Write block 367 len 4096 data: 8 16/30 Read block 224 len 8192 17/30 Read block 199 len 12288 18/30 Read block 114 len 12288 19/30 Read block 71 len 20480 20/30 Read block 490 len 16384 21/30 Write block 444 len 40960 data: 14 14 14 12 11 10 10 7 6 4 22/30 Read block 117 len 12288 23/30 Write block 261 len 36864 data: 7 8 9 9 7 10 13 14 17 24/30 Read block 310 len 20480 25/30 Write block 547 len 24576 data: 20 20 19 20 18 17 26/30 Write block 124 len 32768 data: 16 14 15 14 13 15 15 13 27/30 Write block 281 len 40960 data: 11 11 8 8 8 7 9 8 10 11 28/30 Read block 515 len 20480 29/30 Write block 71 len 32768 data: 8 10 11 12 12 11 9 12 30/30 Write block 188 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 4 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Write 7 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 2 2 31 Done 3 28 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:31.712170964Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:31.712338474Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1734,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 257 257 257 F F F 1 020-039 146 146 146 259 259 259 F F F 2 040-059 144 144 144 257 257 257 F F F 3 060-079 147 147 147 260 260 260 F T T <--- 4 080-099 147 147 147 260 260 260 F F F 5 100-119 144 144 144 257 257 257 F F F 6 120-139 146 148 148 259 262 262 F T T <--- 7 140-159 140 140 140 253 253 253 F F F 8 160-179 146 146 146 259 259 259 F F F 9 180-199 146 146 146 259 259 259 F T T <--- 10 200-219 123 123 123 228 228 228 F T T <--- 11 220-239 146 146 146 259 259 259 F F F 12 240-259 146 148 148 259 262 262 F F F <--- 13 260-279 143 143 143 256 256 256 F T T <--- 14 280-299 141 141 141 254 254 254 F T T <--- 15 300-319 146 146 146 259 259 259 F F F 16 320-339 145 145 145 258 258 258 F F F 17 340-359 146 146 146 259 259 259 F F F 18 360-379 145 145 145 258 258 258 F T T <--- 19 380-399 147 147 147 260 260 260 F F F 20 400-419 144 144 144 257 257 257 F T T <--- 21 420-439 147 148 148 260 261 261 T F F <--- 22 440-459 147 148 148 260 261 261 T T T <--- 23 460-479 142 142 142 255 255 255 F F F 24 480-499 142 142 142 255 255 255 F F F 25 500-519 145 145 145 258 258 258 F F F 26 520-539 145 145 145 258 258 258 F F F 27 540-559 143 148 148 256 261 261 T T T <--- 28 560-579 144 144 144 257 257 257 F F F 29 580-599 146 146 146 259 259 259 F F F Max gen: 148, Max flush: 262 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 257 257 257 F F F 1 020-039 146 146 146 259 259 259 F F F 2 040-059 144 144 144 257 257 257 F F F 3 060-079 149 149 149 263 263 263 F F F 4 080-099 147 147 147 260 260 260 F F F 5 100-119 144 144 144 257 257 257 F F F 6 120-139 149 149 149 263 263 263 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 146 146 146 259 259 259 F F F 9 180-199 149 149 149 263 263 263 F F F 10 200-219 149 149 149 263 263 263 F F F 11 220-239 146 146 146 259 259 259 F F F 12 240-259 148 148 148 262 262 262 F F F 13 260-279 149 149 149 263 263 263 F F F 14 280-299 149 149 149 263 263 263 F F F 15 300-319 146 146 146 259 259 259 F F F 16 320-339 145 145 145 258 258 258 F F F 17 340-359 146 146 146 259 259 259 F F F 18 360-379 149 149 149 263 263 263 F F F 19 380-399 147 147 147 260 260 260 F F F 20 400-419 149 149 149 263 263 263 F F F 21 420-439 148 148 148 261 261 261 F F F 22 440-459 149 149 149 263 263 263 F F F 23 460-479 142 142 142 255 255 255 F F F 24 480-499 142 142 142 255 255 255 F F F 25 500-519 145 145 145 258 258 258 F F F 26 520-539 145 145 145 258 258 258 F F F 27 540-559 149 149 149 263 263 263 F F F 28 560-579 144 144 144 257 257 257 F F F 29 580-599 146 146 146 259 259 259 F F F Max gen: 149, Max flush: 263 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.966761819Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.975091777Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.975368004Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.975394131Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.97540266Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.975414529Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742} {"msg":"Crucible 06901f56-d854-4c41-a840-9e72789404ac has session id: 1744b2f7-4db0-4863-b34f-54ef54f82b8e","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.975735701Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"Upstairs opts: Upstairs UUID: 06901f56-d854-4c41-a840-9e72789404ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.975764498Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"Crucible stats registered with UUID: 06901f56-d854-4c41-a840-9e72789404ac","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.975775507Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:35.975786925Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.975901152Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.97591678Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.975926839Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.9760916Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976106938Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742} {"msg":"06901f56-d854-4c41-a840-9e72789404ac active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976142564Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976152033Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"client":"0","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976161802Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"client":"1","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976171491Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"client":"2","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976181009Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976190948Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976200897Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43003)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976213856Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45099)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976241032Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"ds_connection connected from Ok(127.0.0.1:37624)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976271129Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976880848Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"client":"2","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976907045Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"client":"1","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.976918733Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"client":"0","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977176773Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977186392Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"[0]R flush_numbers[0..12]: [257, 259, 257, 263, 260, 257, 263, 253, 259, 263, 263, 259]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.97720355Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"downstairs","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"[0]R generation[0..12]: [144, 146, 144, 149, 147, 144, 149, 140, 146, 149, 149, 146]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977214489Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"downstairs","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977225518Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"downstairs","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"[1]R flush_numbers[0..12]: [257, 259, 257, 263, 260, 257, 263, 253, 259, 263, 263, 259]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977236046Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"downstairs","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"[1]R generation[0..12]: [144, 146, 144, 149, 147, 144, 149, 140, 146, 149, 149, 146]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977251485Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"downstairs","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977262453Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"downstairs","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"[2]R flush_numbers[0..12]: [257, 259, 257, 263, 260, 257, 263, 253, 259, 263, 263, 259]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977272972Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"downstairs","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"[2]R generation[0..12]: [144, 146, 144, 149, 147, 144, 149, 140, 146, 149, 149, 146]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977283491Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"downstairs","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.97729451Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"downstairs","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977305218Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"downstairs","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977314997Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"downstairs","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977324326Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"downstairs","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977333655Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"":"downstairs","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977343424Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977352313Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"client":"0","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977361922Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"client":"1","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977371671Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"client":"2","":"downstairs","session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977381459Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977390628Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"06901f56-d854-4c41-a840-9e72789404ac is now active with session: 1744b2f7-4db0-4863-b34f-54ef54f82b8e","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977400707Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977409376Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"session_id":"1744b2f7-4db0-4863-b34f-54ef54f82b8e"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977418515Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742} {"msg":"Activated sub_volume 06901f56-d854-4c41-a840-9e72789404ac","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977426424Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977434673Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.977757535Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"task":"crutest"} 01/30 Write block 568 len 32768 data: 13 8 10 13 13 13 14 14 02/30 Write block 333 len 40960 data: 15 13 14 13 14 11 12 11 8 6 03/30 Read block 91 len 4096 04/30 Read block 247 len 36864 05/30 Flush 06/30 Write block 505 len 16384 data: 13 13 13 9 07/30 Write block 353 len 28672 data: 17 18 14 11 8 9 9 08/30 Read block 259 len 36864 09/30 Flush 10/30 Write block 89 len 40960 data: 16 17 16 18 19 19 17 14 9 9 11/30 Read block 220 len 12288 12/30 Write block 64 len 28672 data: 12 12 11 10 10 10 9 13/30 Write block 376 len 8192 data: 12 11 14/30 Write block 82 len 36864 data: 11 14 13 13 15 13 13 17 18 15/30 Read block 166 len 32768 16/30 Write block 360 len 20480 data: 9 8 7 8 9 17/30 Write block 124 len 8192 data: 17 15 18/30 Write block 210 len 8192 data: 8 6 19/30 Write block 128 len 40960 data: 14 16 16 14 13 11 10 11 9 8 20/30 Read block 289 len 20480 21/30 Write block 428 len 32768 data: 11 11 11 13 14 14 15 13 22/30 Read block 376 len 28672 23/30 Write block 92 len 4096 data: 19 24/30 Write block 321 len 8192 data: 16 13 25/30 Write block 68 len 20480 data: 11 11 10 9 11 26/30 Flush 27/30 Write block 489 len 36864 data: 11 13 13 17 16 17 16 15 11 28/30 Read block 190 len 16384 29/30 Write block 268 len 8192 data: 15 18 30/30 Write block 212 len 40960 data: 6 6 4 5 5 6 7 5 7 7 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Sent Done false 1001 Acked Write 10 Done Sent Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 8 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:35.986309147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:35.986482427Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1742,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 257 257 257 F F F 1 020-039 146 146 146 259 259 259 F F F 2 040-059 144 144 144 257 257 257 F F F 3 060-079 150 149 150 266 263 266 F F F <--- 4 080-099 150 147 150 266 260 266 F F F <--- 5 100-119 144 144 144 257 257 257 F F F 6 120-139 150 149 150 266 263 266 F F F <--- 7 140-159 140 140 140 253 253 253 F F F 8 160-179 146 146 146 259 259 259 F F F 9 180-199 149 149 149 263 263 263 F F F 10 200-219 150 149 150 266 263 266 T F T <--- 11 220-239 146 146 146 259 259 259 T F T <--- 12 240-259 148 148 148 262 262 262 F F F 13 260-279 149 149 149 263 263 263 T F T <--- 14 280-299 149 149 149 263 263 263 F F F 15 300-319 146 146 146 259 259 259 F F F 16 320-339 150 145 150 266 258 266 F F F <--- 17 340-359 150 146 150 265 259 265 F F F <--- 18 360-379 150 149 150 266 263 266 F F F <--- 19 380-399 147 147 147 260 260 260 F F F 20 400-419 149 149 149 263 263 263 F F F 21 420-439 150 148 150 266 261 266 F F F <--- 22 440-459 149 149 149 263 263 263 F F F 23 460-479 142 142 142 255 255 255 F F F 24 480-499 142 142 142 255 255 255 T F T <--- 25 500-519 150 145 150 265 258 265 F F F <--- 26 520-539 145 145 145 258 258 258 F F F 27 540-559 149 149 149 263 263 263 F F F 28 560-579 150 144 150 264 257 264 F F F <--- 29 580-599 146 146 146 259 259 259 F F F Max gen: 150, Max flush: 266 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 257 257 257 F F F 1 020-039 146 146 146 259 259 259 F F F 2 040-059 144 144 144 257 257 257 F F F 3 060-079 150 150 150 266 266 266 F F F 4 080-099 150 150 150 266 266 266 F F F 5 100-119 144 144 144 257 257 257 F F F 6 120-139 150 150 150 266 266 266 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 146 146 146 259 259 259 F F F 9 180-199 149 149 149 263 263 263 F F F 10 200-219 151 151 151 267 267 267 F F F 11 220-239 151 151 151 267 267 267 F F F 12 240-259 148 148 148 262 262 262 F F F 13 260-279 151 151 151 267 267 267 F F F 14 280-299 149 149 149 263 263 263 F F F 15 300-319 146 146 146 259 259 259 F F F 16 320-339 150 150 150 266 266 266 F F F 17 340-359 150 150 150 265 265 265 F F F 18 360-379 150 150 150 266 266 266 F F F 19 380-399 147 147 147 260 260 260 F F F 20 400-419 149 149 149 263 263 263 F F F 21 420-439 150 150 150 266 266 266 F F F 22 440-459 149 149 149 263 263 263 F F F 23 460-479 142 142 142 255 255 255 F F F 24 480-499 151 151 151 267 267 267 F F F 25 500-519 150 150 150 265 265 265 F F F 26 520-539 145 145 145 258 258 258 F F F 27 540-559 149 149 149 263 263 263 F F F 28 560-579 150 150 150 264 264 264 F F F 29 580-599 146 146 146 259 259 259 F F F Max gen: 151, Max flush: 267 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.23904708Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.247209337Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.247487904Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.247515791Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.24752447Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.247536079Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750} {"msg":"Crucible 4f90e5a6-f34c-4be0-a210-18a49b6a83dd has session id: e240f47a-15e3-4540-91cf-df1b210ff8ea","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.247862151Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"Upstairs opts: Upstairs UUID: 4f90e5a6-f34c-4be0-a210-18a49b6a83dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.247884078Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"Crucible stats registered with UUID: 4f90e5a6-f34c-4be0-a210-18a49b6a83dd","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.247894297Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:40.247903416Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.248056698Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.248084245Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.248095253Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.248228718Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.248248266Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750} {"msg":"4f90e5a6-f34c-4be0-a210-18a49b6a83dd active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.248274562Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.248284041Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"client":"0","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.24829373Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"client":"1","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.248303679Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"client":"2","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.248313228Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.248332496Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.248371321Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:51548)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.24838234Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:37266)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.248392709Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:40086)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.24846906Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249045152Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"client":"1","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.24906311Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"client":"0","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249074409Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"client":"2","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249371504Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249395332Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"[0]R flush_numbers[0..12]: [257, 259, 257, 266, 266, 257, 266, 253, 259, 263, 267, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.24940926Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"downstairs","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"[0]R generation[0..12]: [144, 146, 144, 150, 150, 144, 150, 140, 146, 149, 151, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249420919Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"downstairs","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249432147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"downstairs","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"[1]R flush_numbers[0..12]: [257, 259, 257, 266, 266, 257, 266, 253, 259, 263, 267, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249442876Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"downstairs","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"[1]R generation[0..12]: [144, 146, 144, 150, 150, 144, 150, 140, 146, 149, 151, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249453465Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"downstairs","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249464444Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"downstairs","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"[2]R flush_numbers[0..12]: [257, 259, 257, 266, 266, 257, 266, 253, 259, 263, 267, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249475152Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"downstairs","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"[2]R generation[0..12]: [144, 146, 144, 150, 150, 144, 150, 140, 146, 149, 151, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249485761Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"downstairs","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.2494966Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"downstairs","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249506249Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"downstairs","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249515938Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"downstairs","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249525306Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"downstairs","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249534635Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"":"downstairs","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249544264Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249554483Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"client":"0","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249564032Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"client":"1","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249573641Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"client":"2","":"downstairs","session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.24958329Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249592359Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"4f90e5a6-f34c-4be0-a210-18a49b6a83dd is now active with session: e240f47a-15e3-4540-91cf-df1b210ff8ea","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249602187Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249610836Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"session_id":"e240f47a-15e3-4540-91cf-df1b210ff8ea"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249619985Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750} {"msg":"Activated sub_volume 4f90e5a6-f34c-4be0-a210-18a49b6a83dd","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249628004Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249636034Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.249929289Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"task":"crutest"} 01/30 Write block 479 len 32768 data: 13 9 12 10 9 11 9 10 02/30 Read block 589 len 16384 03/30 Write block 389 len 40960 data: 10 9 11 10 11 14 13 13 12 14 04/30 Read block 132 len 28672 05/30 Write block 201 len 24576 data: 8 10 11 11 10 9 06/30 Read block 499 len 40960 07/30 Read block 138 len 32768 08/30 Write block 26 len 32768 data: 17 15 18 18 19 20 19 17 09/30 Read block 246 len 40960 10/30 Read block 463 len 28672 11/30 Write block 360 len 40960 data: 10 9 8 9 10 8 8 9 9 7 12/30 Read block 72 len 8192 13/30 Read block 298 len 20480 14/30 Write block 68 len 16384 data: 12 12 11 10 15/30 Read block 246 len 32768 16/30 Flush 17/30 Write block 96 len 4096 data: 15 18/30 Read block 74 len 32768 19/30 Read block 328 len 28672 20/30 Write block 466 len 20480 data: 6 6 5 5 3 21/30 Read block 422 len 28672 22/30 Read block 322 len 24576 23/30 Read block 344 len 40960 24/30 Read block 39 len 40960 25/30 Read block 397 len 20480 26/30 Flush 27/30 Write block 43 len 24576 data: 11 12 13 14 13 10 28/30 Read block 232 len 24576 29/30 Read block 593 len 28672 30/30 Read block 587 len 32768 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 4 Done Done Sent false 1002 Acked Write 10 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Read 10 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Write 4 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Read 10 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 30 29 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:40.260237466Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:40.260420715Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1750,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 257 257 257 F F F 1 020-039 152 152 146 268 268 259 F F F <--- 2 040-059 144 144 144 257 257 257 T T F <--- 3 060-079 152 152 150 268 268 266 F F F <--- 4 080-099 152 152 150 269 269 266 F F F <--- 5 100-119 144 144 144 257 257 257 F F F 6 120-139 150 150 150 266 266 266 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 146 146 146 259 259 259 F F F 9 180-199 149 149 149 263 263 263 F F F 10 200-219 152 152 151 268 268 267 F F F <--- 11 220-239 151 151 151 267 267 267 F F F 12 240-259 148 148 148 262 262 262 F F F 13 260-279 151 151 151 267 267 267 F F F 14 280-299 149 149 149 263 263 263 F F F 15 300-319 146 146 146 259 259 259 F F F 16 320-339 150 150 150 266 266 266 F F F 17 340-359 150 150 150 265 265 265 F F F 18 360-379 152 152 150 268 268 266 F F F <--- 19 380-399 152 152 147 268 268 260 F F F <--- 20 400-419 149 149 149 263 263 263 F F F 21 420-439 150 150 150 266 266 266 F F F 22 440-459 149 149 149 263 263 263 F F F 23 460-479 152 152 142 269 269 255 F F T <--- 24 480-499 152 152 151 268 268 267 F F T <--- 25 500-519 150 150 150 265 265 265 F F F 26 520-539 145 145 145 258 258 258 F F F 27 540-559 149 149 149 263 263 263 F F F 28 560-579 150 150 150 264 264 264 F F F 29 580-599 146 146 146 259 259 259 F F F Max gen: 152, Max flush: 269 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 144 144 144 257 257 257 F F F 1 020-039 152 152 152 268 268 268 F F F 2 040-059 153 153 153 270 270 270 F F F 3 060-079 152 152 152 268 268 268 F F F 4 080-099 152 152 152 269 269 269 F F F 5 100-119 144 144 144 257 257 257 F F F 6 120-139 150 150 150 266 266 266 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 146 146 146 259 259 259 F F F 9 180-199 149 149 149 263 263 263 F F F 10 200-219 152 152 152 268 268 268 F F F 11 220-239 151 151 151 267 267 267 F F F 12 240-259 148 148 148 262 262 262 F F F 13 260-279 151 151 151 267 267 267 F F F 14 280-299 149 149 149 263 263 263 F F F 15 300-319 146 146 146 259 259 259 F F F 16 320-339 150 150 150 266 266 266 F F F 17 340-359 150 150 150 265 265 265 F F F 18 360-379 152 152 152 268 268 268 F F F 19 380-399 152 152 152 268 268 268 F F F 20 400-419 149 149 149 263 263 263 F F F 21 420-439 150 150 150 266 266 266 F F F 22 440-459 149 149 149 263 263 263 F F F 23 460-479 152 152 152 269 269 269 F F F 24 480-499 152 152 152 268 268 268 F F F 25 500-519 150 150 150 265 265 265 F F F 26 520-539 145 145 145 258 258 258 F F F 27 540-559 149 149 149 263 263 263 F F F 28 560-579 150 150 150 264 264 264 F F F 29 580-599 146 146 146 259 259 259 F F F Max gen: 153, Max flush: 270 Begin loop 77 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.502307884Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.510378362Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.510667668Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.510703294Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.510712612Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.510724231Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758} {"msg":"Crucible 8aa8523e-6a2e-47ff-a80f-35711e20179f has session id: dde527e1-b89f-40b3-a797-405f08c3b9d5","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511044334Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"Upstairs opts: Upstairs UUID: 8aa8523e-6a2e-47ff-a80f-35711e20179f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.51107381Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"Crucible stats registered with UUID: 8aa8523e-6a2e-47ff-a80f-35711e20179f","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511085029Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:44.511094408Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511234712Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511264248Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511275267Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511407181Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511426309Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758} {"msg":"8aa8523e-6a2e-47ff-a80f-35711e20179f active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511449516Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511459005Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"client":"0","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511468634Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"client":"1","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511478373Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"client":"2","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511488002Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511497791Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511509519Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:47029)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511559554Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58634)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511570322Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:50541)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.511580551Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512248173Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"client":"2","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512269281Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"client":"0","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512280469Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"client":"1","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512557637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512567326Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"[0]R flush_numbers[0..12]: [257, 268, 270, 268, 269, 257, 266, 253, 259, 263, 268, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512596653Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"downstairs","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"[0]R generation[0..12]: [144, 152, 153, 152, 152, 144, 150, 140, 146, 149, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512608071Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"downstairs","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.51261913Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"downstairs","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"[1]R flush_numbers[0..12]: [257, 268, 270, 268, 269, 257, 266, 253, 259, 263, 268, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512630029Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"downstairs","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"[1]R generation[0..12]: [144, 152, 153, 152, 152, 144, 150, 140, 146, 149, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512640637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"downstairs","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512653966Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"downstairs","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"[2]R flush_numbers[0..12]: [257, 268, 270, 268, 269, 257, 266, 253, 259, 263, 268, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512664515Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"downstairs","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"[2]R generation[0..12]: [144, 152, 153, 152, 152, 144, 150, 140, 146, 149, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512675183Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"downstairs","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512686162Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"downstairs","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512695671Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"downstairs","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.51270555Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"downstairs","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512715109Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"downstairs","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512724638Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"":"downstairs","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512734406Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512743315Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"client":"0","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512752884Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"client":"1","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512762513Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"client":"2","":"downstairs","session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512772392Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512781521Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"8aa8523e-6a2e-47ff-a80f-35711e20179f is now active with session: dde527e1-b89f-40b3-a797-405f08c3b9d5","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.51279145Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512801469Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"session_id":"dde527e1-b89f-40b3-a797-405f08c3b9d5"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512810628Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758} {"msg":"Activated sub_volume 8aa8523e-6a2e-47ff-a80f-35711e20179f","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512818677Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.512826866Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.513139119Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"task":"crutest"} 01/30 Write block 10 len 40960 data: 14 13 13 12 9 8 8 10 9 11 02/30 Write block 385 len 36864 data: 8 10 9 8 11 10 12 11 12 03/30 Read block 293 len 40960 04/30 Flush 05/30 Write block 56 len 32768 data: 20 16 16 13 13 14 12 11 06/30 Read block 167 len 36864 07/30 Read block 332 len 4096 08/30 Read block 507 len 4096 09/30 Read block 420 len 36864 10/30 Write block 296 len 28672 data: 14 15 15 13 17 17 17 11/30 Read block 481 len 20480 12/30 Write block 166 len 40960 data: 12 10 12 12 9 11 11 11 7 7 13/30 Flush 14/30 Write block 383 len 16384 data: 13 10 9 11 15/30 Read block 581 len 40960 16/30 Read block 450 len 12288 17/30 Flush 18/30 Flush 19/30 Write block 39 len 36864 data: 11 13 12 11 12 13 14 15 14 20/30 Read block 445 len 32768 21/30 Read block 202 len 16384 22/30 Read block 193 len 28672 23/30 Read block 244 len 16384 24/30 Read block 545 len 20480 25/30 Write block 437 len 16384 data: 10 11 14 13 26/30 Read block 142 len 12288 27/30 Write block 284 len 28672 data: 9 9 8 10 9 11 12 28/30 Read block 255 len 40960 29/30 Read block 317 len 20480 30/30 Read block 50 len 24576 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 0 27 Done 30 3 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:44.523182537Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:44.523360166Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1758,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 144 154 271 257 271 F T F <--- 1 020-039 152 152 152 268 268 268 T F T <--- 2 040-059 154 153 154 272 270 272 T F T <--- 3 060-079 154 152 154 272 268 272 F F F <--- 4 080-099 152 152 152 269 269 269 F F F 5 100-119 144 144 144 257 257 257 F F F 6 120-139 150 150 150 266 266 266 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 154 146 154 272 259 272 F F F <--- 9 180-199 149 149 149 263 263 263 F F F 10 200-219 152 152 152 268 268 268 F F F 11 220-239 151 151 151 267 267 267 F F F 12 240-259 148 148 148 262 262 262 F F F 13 260-279 151 151 151 267 267 267 F F F 14 280-299 154 149 154 272 263 272 T F T <--- 15 300-319 154 146 154 272 259 272 F F F <--- 16 320-339 150 150 150 266 266 266 F F F 17 340-359 150 150 150 265 265 265 F F F 18 360-379 152 152 152 268 268 268 F F F 19 380-399 154 152 154 273 268 273 F T F <--- 20 400-419 149 149 149 263 263 263 F F F 21 420-439 150 150 150 266 266 266 T F T <--- 22 440-459 149 149 149 263 263 263 T F T <--- 23 460-479 152 152 152 269 269 269 F F F 24 480-499 152 152 152 268 268 268 F F F 25 500-519 150 150 150 265 265 265 F F F 26 520-539 145 145 145 258 258 258 F F F 27 540-559 149 149 149 263 263 263 F F F 28 560-579 150 150 150 264 264 264 F F F 29 580-599 146 146 146 259 259 259 F F F Max gen: 154, Max flush: 273 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 271 271 271 F F F 1 020-039 155 155 155 274 274 274 F F F 2 040-059 155 155 155 274 274 274 F F F 3 060-079 154 154 154 272 272 272 F F F 4 080-099 152 152 152 269 269 269 F F F 5 100-119 144 144 144 257 257 257 F F F 6 120-139 150 150 150 266 266 266 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 154 154 154 272 272 272 F F F 9 180-199 149 149 149 263 263 263 F F F 10 200-219 152 152 152 268 268 268 F F F 11 220-239 151 151 151 267 267 267 F F F 12 240-259 148 148 148 262 262 262 F F F 13 260-279 151 151 151 267 267 267 F F F 14 280-299 155 155 155 274 274 274 F F F 15 300-319 154 154 154 272 272 272 F F F 16 320-339 150 150 150 266 266 266 F F F 17 340-359 150 150 150 265 265 265 F F F 18 360-379 152 152 152 268 268 268 F F F 19 380-399 154 154 154 273 273 273 F F F 20 400-419 149 149 149 263 263 263 F F F 21 420-439 155 155 155 274 274 274 F F F 22 440-459 155 155 155 274 274 274 F F F 23 460-479 152 152 152 269 269 269 F F F 24 480-499 152 152 152 268 268 268 F F F 25 500-519 150 150 150 265 265 265 F F F 26 520-539 145 145 145 258 258 258 F F F 27 540-559 149 149 149 263 263 263 F F F 28 560-579 150 150 150 264 264 264 F F F 29 580-599 146 146 146 259 259 259 F F F Max gen: 155, Max flush: 274 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.770446218Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.778454243Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.778730961Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.778754968Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.778763697Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.778775316Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766} {"msg":"Crucible f3cceadc-c2f8-4fcf-903a-2df76a1beac7 has session id: 65aa3c85-a8bd-4e25-ba82-a0628f84fe28","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779124415Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"Upstairs opts: Upstairs UUID: f3cceadc-c2f8-4fcf-903a-2df76a1beac7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779151962Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"Crucible stats registered with UUID: f3cceadc-c2f8-4fcf-903a-2df76a1beac7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779162821Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:48.7791721Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779317533Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779345679Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779356408Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779486133Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779501871Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766} {"msg":"f3cceadc-c2f8-4fcf-903a-2df76a1beac7 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779519159Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779528178Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"client":"0","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779538027Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"client":"1","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779547656Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"client":"2","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779557195Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779567093Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.77959828Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"ds_connection connected from Ok(127.0.0.1:47771)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779641605Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"ds_connection connected from Ok(127.0.0.1:37428)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779657933Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"ds_connection connected from Ok(127.0.0.1:39365)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.779668882Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780390647Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"client":"0","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780412245Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"client":"2","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780423494Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"client":"1","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780717099Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780738277Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"[0]R flush_numbers[0..12]: [271, 274, 274, 272, 269, 257, 266, 253, 272, 263, 268, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780748786Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"downstairs","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"[0]R generation[0..12]: [154, 155, 155, 154, 152, 144, 150, 140, 154, 149, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780759674Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"downstairs","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780770713Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"downstairs","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"[1]R flush_numbers[0..12]: [271, 274, 274, 272, 269, 257, 266, 253, 272, 263, 268, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780781272Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"downstairs","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"[1]R generation[0..12]: [154, 155, 155, 154, 152, 144, 150, 140, 154, 149, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780791911Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"downstairs","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780802829Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"downstairs","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"[2]R flush_numbers[0..12]: [271, 274, 274, 272, 269, 257, 266, 253, 272, 263, 268, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780813468Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"downstairs","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"[2]R generation[0..12]: [154, 155, 155, 154, 152, 144, 150, 140, 154, 149, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780824127Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"downstairs","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780835045Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"downstairs","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780844504Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"downstairs","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780857073Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"downstairs","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780866472Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"downstairs","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780875851Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"":"downstairs","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.78088593Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780894998Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"client":"0","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780904847Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"client":"1","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780914476Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"client":"2","":"downstairs","session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780924295Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780933514Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"f3cceadc-c2f8-4fcf-903a-2df76a1beac7 is now active with session: 65aa3c85-a8bd-4e25-ba82-a0628f84fe28","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780943493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780952312Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"session_id":"65aa3c85-a8bd-4e25-ba82-a0628f84fe28"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780961541Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766} {"msg":"Activated sub_volume f3cceadc-c2f8-4fcf-903a-2df76a1beac7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.78096951Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.780977769Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.781269545Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"task":"crutest"} 01/30 Flush 02/30 Flush 03/30 Write block 426 len 4096 data: 9 04/30 Write block 202 len 32768 data: 11 12 12 11 10 8 11 14 05/30 Flush 06/30 Write block 298 len 32768 data: 16 14 18 18 18 18 15 19 07/30 Read block 197 len 20480 08/30 Read block 359 len 28672 09/30 Read block 29 len 36864 10/30 Read block 532 len 28672 11/30 Write block 396 len 36864 data: 14 13 15 13 14 11 8 8 10 12/30 Write block 596 len 12288 data: 9 9 8 13/30 Write block 479 len 12288 data: 14 10 13 14/30 Flush 15/30 Write block 23 len 36864 data: 11 14 13 18 16 19 19 20 21 16/30 Read block 315 len 24576 17/30 Read block 355 len 28672 18/30 Read block 139 len 16384 19/30 Read block 377 len 24576 20/30 Read block 397 len 40960 21/30 Write block 254 len 20480 data: 11 10 11 12 6 22/30 Write block 551 len 36864 data: 19 18 18 17 19 17 13 14 14 23/30 Read block 458 len 32768 24/30 Read block 32 len 12288 25/30 Read block 370 len 28672 26/30 Write block 500 len 32768 data: 13 11 9 11 11 14 14 14 27/30 Read block 143 len 8192 28/30 Write block 417 len 24576 data: 11 10 11 11 12 9 29/30 Read block 492 len 32768 30/30 Write block 170 len 12288 data: 10 12 12 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 1 Done Done Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 27 1 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1001)) Some(JobId(1013)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:48.790642911Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:48.790824729Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1766,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 271 271 271 F F F 1 020-039 155 155 155 274 274 274 T F T <--- 2 040-059 155 155 155 274 274 274 F F F 3 060-079 154 154 154 272 272 272 F F F 4 080-099 152 152 152 269 269 269 F F F 5 100-119 144 144 144 257 257 257 F F F 6 120-139 150 150 150 266 266 266 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 154 154 154 272 272 272 T F T <--- 9 180-199 149 149 149 263 263 263 F F F 10 200-219 156 152 156 277 268 277 F F F <--- 11 220-239 151 151 151 267 267 267 F F F 12 240-259 148 148 148 262 262 262 T F T <--- 13 260-279 151 151 151 267 267 267 F F F 14 280-299 156 155 156 278 274 278 F F F <--- 15 300-319 156 154 156 278 272 278 F F F <--- 16 320-339 150 150 150 266 266 266 F F F 17 340-359 150 150 150 265 265 265 F F F 18 360-379 152 152 152 268 268 268 F F F 19 380-399 156 154 156 278 273 278 F F F <--- 20 400-419 156 149 156 278 263 278 T F T <--- 21 420-439 156 155 156 277 274 277 T T T <--- 22 440-459 155 155 155 274 274 274 F F F 23 460-479 156 152 156 278 269 278 F F F <--- 24 480-499 156 152 156 278 268 278 F F F <--- 25 500-519 150 150 150 265 265 265 T F T <--- 26 520-539 145 145 145 258 258 258 F F F 27 540-559 149 149 149 263 263 263 T F T <--- 28 560-579 150 150 150 264 264 264 F F F 29 580-599 156 146 156 278 259 278 F F F <--- Max gen: 156, Max flush: 278 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 271 271 271 F F F 1 020-039 157 157 157 279 279 279 F F F 2 040-059 155 155 155 274 274 274 F F F 3 060-079 154 154 154 272 272 272 F F F 4 080-099 152 152 152 269 269 269 F F F 5 100-119 144 144 144 257 257 257 F F F 6 120-139 150 150 150 266 266 266 F F F 7 140-159 140 140 140 253 253 253 F F F 8 160-179 157 157 157 279 279 279 F F F 9 180-199 149 149 149 263 263 263 F F F 10 200-219 156 156 156 277 277 277 F F F 11 220-239 151 151 151 267 267 267 F F F 12 240-259 157 157 157 279 279 279 F F F 13 260-279 151 151 151 267 267 267 F F F 14 280-299 156 156 156 278 278 278 F F F 15 300-319 156 156 156 278 278 278 F F F 16 320-339 150 150 150 266 266 266 F F F 17 340-359 150 150 150 265 265 265 F F F 18 360-379 152 152 152 268 268 268 F F F 19 380-399 156 156 156 278 278 278 F F F 20 400-419 157 157 157 279 279 279 F F F 21 420-439 157 157 157 279 279 279 F F F 22 440-459 155 155 155 274 274 274 F F F 23 460-479 156 156 156 278 278 278 F F F 24 480-499 156 156 156 278 278 278 F F F 25 500-519 157 157 157 279 279 279 F F F 26 520-539 145 145 145 258 258 258 F F F 27 540-559 157 157 157 279 279 279 F F F 28 560-579 150 150 150 264 264 264 F F F 29 580-599 156 156 156 278 278 278 F F F Max gen: 157, Max flush: 279 Begin loop 79 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.043037162Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.051177102Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.051449361Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.051478717Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.051487066Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.051498505Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774} {"msg":"Crucible f8997e80-ed90-481b-ada3-3c24be1f25ba has session id: 32c4de44-4648-432f-9911-fed1980d9bb6","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.051860723Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"Upstairs opts: Upstairs UUID: f8997e80-ed90-481b-ada3-3c24be1f25ba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.051889859Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"Crucible stats registered with UUID: f8997e80-ed90-481b-ada3-3c24be1f25ba","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.051900638Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:53.051909757Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052043431Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052074348Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052085606Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052180895Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052194584Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774} {"msg":"f8997e80-ed90-481b-ada3-3c24be1f25ba active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052230509Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052242738Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"client":"0","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052252767Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"client":"1","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052262606Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"client":"2","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052272154Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052282003Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052291932Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:50588)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052359364Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:58437)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052443155Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:46447)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.052464872Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053008868Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"client":"0","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053026336Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"client":"1","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053037325Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"client":"2","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053346749Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053390734Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"[0]R flush_numbers[0..12]: [271, 279, 274, 272, 269, 257, 266, 253, 279, 263, 277, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053439678Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"downstairs","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"[0]R generation[0..12]: [154, 157, 155, 154, 152, 144, 150, 140, 157, 149, 156, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053453437Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"downstairs","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053464625Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"downstairs","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"[1]R flush_numbers[0..12]: [271, 279, 274, 272, 269, 257, 266, 253, 279, 263, 277, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053475374Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"downstairs","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"[1]R generation[0..12]: [154, 157, 155, 154, 152, 144, 150, 140, 157, 149, 156, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053486073Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"downstairs","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053496992Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"downstairs","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"[2]R flush_numbers[0..12]: [271, 279, 274, 272, 269, 257, 266, 253, 279, 263, 277, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.05350761Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"downstairs","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"[2]R generation[0..12]: [154, 157, 155, 154, 152, 144, 150, 140, 157, 149, 156, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053518469Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"downstairs","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053529458Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"downstairs","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053538977Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"downstairs","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053548705Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"downstairs","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053558114Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"downstairs","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053567483Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"":"downstairs","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053577232Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053585941Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"client":"0","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.05359555Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"client":"1","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053606719Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"client":"2","":"downstairs","session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053616708Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053625866Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"f8997e80-ed90-481b-ada3-3c24be1f25ba is now active with session: 32c4de44-4648-432f-9911-fed1980d9bb6","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053635795Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053644504Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"session_id":"32c4de44-4648-432f-9911-fed1980d9bb6"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053653553Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774} {"msg":"Activated sub_volume f8997e80-ed90-481b-ada3-3c24be1f25ba","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053661572Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053669721Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.053966437Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"task":"crutest"} 01/30 Write block 416 len 36864 data: 12 12 11 12 12 13 10 8 6 02/30 Write block 441 len 40960 data: 17 16 13 15 15 15 13 12 11 11 03/30 Read block 558 len 24576 04/30 Read block 133 len 20480 05/30 Write block 16 len 24576 data: 9 11 10 12 11 10 06/30 Write block 271 len 16384 data: 14 13 14 14 07/30 Read block 538 len 4096 08/30 Write block 21 len 32768 data: 11 12 12 15 14 19 17 20 09/30 Read block 344 len 28672 10/30 Read block 460 len 36864 11/30 Write block 259 len 4096 data: 6 12/30 Write block 415 len 12288 data: 11 13 13 13/30 Read block 460 len 8192 14/30 Write block 503 len 8192 data: 12 12 15/30 Read block 413 len 36864 16/30 Write block 88 len 12288 data: 14 18 19 17/30 Read block 424 len 28672 18/30 Write block 0 len 32768 data: 4 7 10 10 14 15 16 16 19/30 Flush 20/30 Write block 62 len 36864 data: 13 12 13 13 12 11 13 13 12 21/30 Write block 149 len 32768 data: 13 14 16 16 16 14 14 13 22/30 Write block 143 len 32768 data: 13 14 15 13 15 16 14 15 23/30 Read block 353 len 28672 24/30 Read block 0 len 36864 25/30 Write block 562 len 12288 data: 11 12 14 26/30 Write block 518 len 32768 data: 15 14 12 12 12 13 12 10 27/30 Read block 50 len 12288 28/30 Write block 411 len 40960 data: 11 12 12 12 12 14 14 12 13 13 29/30 Read block 230 len 28672 30/30 Write block 334 len 16384 data: 14 15 14 15 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 10 Done Done Sent false 1002 Acked Read 6 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Write 4 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Write 3 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:53.063169113Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:53.063394476Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1774,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 154 280 280 271 F F F <--- 1 020-039 158 158 157 280 280 279 F F F <--- 2 040-059 155 155 155 274 274 274 F F F 3 060-079 154 154 154 272 272 272 T T F <--- 4 080-099 158 158 152 280 280 269 F F F <--- 5 100-119 144 144 144 257 257 257 F F F 6 120-139 150 150 150 266 266 266 F F F 7 140-159 140 140 140 253 253 253 T T F <--- 8 160-179 157 157 157 279 279 279 F F F 9 180-199 149 149 149 263 263 263 F F F 10 200-219 156 156 156 277 277 277 F F F 11 220-239 151 151 151 267 267 267 F F F 12 240-259 158 158 157 280 280 279 F F F <--- 13 260-279 158 158 151 280 280 267 F F F <--- 14 280-299 156 156 156 278 278 278 F F F 15 300-319 156 156 156 278 278 278 F F F 16 320-339 150 150 150 266 266 266 T T F <--- 17 340-359 150 150 150 265 265 265 F F F 18 360-379 152 152 152 268 268 268 F F F 19 380-399 156 156 156 278 278 278 F F F 20 400-419 158 158 157 280 280 279 T T T <--- 21 420-439 158 158 157 280 280 279 T T T <--- 22 440-459 158 158 155 280 280 274 F F F <--- 23 460-479 156 156 156 278 278 278 F F F 24 480-499 156 156 156 278 278 278 F F F 25 500-519 158 158 157 280 280 279 T T F <--- 26 520-539 145 145 145 258 258 258 T T F <--- 27 540-559 157 157 157 279 279 279 F F F 28 560-579 150 150 150 264 264 264 T T F <--- 29 580-599 156 156 156 278 278 278 F F F Max gen: 158, Max flush: 280 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 280 280 280 F F F 1 020-039 158 158 158 280 280 280 F F F 2 040-059 155 155 155 274 274 274 F F F 3 060-079 159 159 159 281 281 281 F F F 4 080-099 158 158 158 280 280 280 F F F 5 100-119 144 144 144 257 257 257 F F F 6 120-139 150 150 150 266 266 266 F F F 7 140-159 159 159 159 281 281 281 F F F 8 160-179 157 157 157 279 279 279 F F F 9 180-199 149 149 149 263 263 263 F F F 10 200-219 156 156 156 277 277 277 F F F 11 220-239 151 151 151 267 267 267 F F F 12 240-259 158 158 158 280 280 280 F F F 13 260-279 158 158 158 280 280 280 F F F 14 280-299 156 156 156 278 278 278 F F F 15 300-319 156 156 156 278 278 278 F F F 16 320-339 159 159 159 281 281 281 F F F 17 340-359 150 150 150 265 265 265 F F F 18 360-379 152 152 152 268 268 268 F F F 19 380-399 156 156 156 278 278 278 F F F 20 400-419 159 159 159 281 281 281 F F F 21 420-439 159 159 159 281 281 281 F F F 22 440-459 158 158 158 280 280 280 F F F 23 460-479 156 156 156 278 278 278 F F F 24 480-499 156 156 156 278 278 278 F F F 25 500-519 159 159 159 281 281 281 F F F 26 520-539 159 159 159 281 281 281 F F F 27 540-559 157 157 157 279 279 279 F F F 28 560-579 159 159 159 281 281 281 F F F 29 580-599 156 156 156 278 278 278 F F F Max gen: 159, Max flush: 281 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.324448728Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.332604786Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.332878334Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.332904541Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.33291316Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.332924848Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782} {"msg":"Crucible 8badde73-7df3-43d4-ac59-fac9a708371c has session id: 5b063f12-de53-4b64-885e-2052f3f36c00","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333204116Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"Upstairs opts: Upstairs UUID: 8badde73-7df3-43d4-ac59-fac9a708371c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333236662Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"Crucible stats registered with UUID: 8badde73-7df3-43d4-ac59-fac9a708371c","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333248001Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:57.333257399Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.33342417Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333452507Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333489942Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333606199Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333620047Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782} {"msg":"8badde73-7df3-43d4-ac59-fac9a708371c active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333644284Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333654023Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"client":"0","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333663742Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"client":"1","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333673471Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"client":"2","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.33368303Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333693369Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333738033Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"ds_connection connected from Ok(127.0.0.1:61622)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333748952Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"ds_connection connected from Ok(127.0.0.1:64965)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333759331Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"ds_connection connected from Ok(127.0.0.1:49579)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.333829623Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334408315Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"client":"2","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334435982Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"client":"1","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334462389Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"client":"0","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334753345Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334776142Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"[0]R flush_numbers[0..12]: [280, 280, 274, 281, 280, 257, 266, 281, 279, 263, 277, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334841015Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"downstairs","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"[0]R generation[0..12]: [158, 158, 155, 159, 158, 144, 150, 159, 157, 149, 156, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334852883Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"downstairs","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334864102Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"downstairs","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"[1]R flush_numbers[0..12]: [280, 280, 274, 281, 280, 257, 266, 281, 279, 263, 277, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334874681Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"downstairs","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"[1]R generation[0..12]: [158, 158, 155, 159, 158, 144, 150, 159, 157, 149, 156, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334885359Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"downstairs","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334898918Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"downstairs","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"[2]R flush_numbers[0..12]: [280, 280, 274, 281, 280, 257, 266, 281, 279, 263, 277, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334909637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"downstairs","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"[2]R generation[0..12]: [158, 158, 155, 159, 158, 144, 150, 159, 157, 149, 156, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334920235Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"downstairs","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334931084Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"downstairs","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334940583Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"downstairs","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334950302Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"downstairs","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334959671Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"downstairs","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.33496897Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"":"downstairs","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334978879Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334987628Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"client":"0","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.334997446Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"client":"1","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.335007075Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"client":"2","":"downstairs","session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.335016844Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"Set Downstairs and Upstairs active after reconciliation","vDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.335031602Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"8badde73-7df3-43d4-ac59-fac9a708371c is now active with session: 5b063f12-de53-4b64-885e-2052f3f36c00","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.335041641Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.33505186Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"session_id":"5b063f12-de53-4b64-885e-2052f3f36c00"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.335060959Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782} {"msg":"Activated sub_volume 8badde73-7df3-43d4-ac59-fac9a708371c","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.335069078Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.335077157Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.335289282Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"task":"crutest"} 01/30 Write block 189 len 8192 data: 13 15 02/30 Write block 403 len 36864 data: 9 11 7 7 9 9 10 11 12 03/30 Read block 58 len 24576 04/30 Write block 464 len 24576 data: 8 7 7 7 6 6 05/30 Write block 316 len 12288 data: 10 13 14 06/30 Write block 360 len 40960 data: 11 10 9 10 11 9 9 10 10 8 07/30 Read block 412 len 4096 08/30 Write block 546 len 40960 data: 18 21 21 20 21 20 19 19 18 20 09/30 Read block 501 len 16384 10/30 Read block 246 len 24576 11/30 Read block 169 len 8192 12/30 Read block 11 len 12288 13/30 Read block 56 len 20480 14/30 Write block 366 len 20480 data: 10 11 11 9 6 15/30 Write block 270 len 40960 data: 17 15 14 15 15 16 15 13 14 15 16/30 Write block 284 len 24576 data: 10 10 9 11 10 12 17/30 Read block 383 len 12288 18/30 Write block 561 len 40960 data: 11 12 13 15 12 11 12 14 9 11 19/30 Read block 158 len 24576 20/30 Read block 524 len 20480 21/30 Write block 538 len 20480 data: 12 12 10 11 12 22/30 Write block 267 len 28672 data: 14 16 19 18 16 15 16 23/30 Flush 24/30 Write block 527 len 20480 data: 6 7 9 12 12 25/30 Read block 358 len 20480 26/30 Read block 299 len 24576 27/30 Read block 322 len 32768 28/30 Write block 552 len 20480 data: 20 20 19 21 18 29/30 Read block 331 len 36864 30/30 Write block 405 len 36864 data: 8 8 10 10 11 12 13 13 13 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Write 6 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 10 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Read 5 Done Sent Done false 1025 Acked Read 6 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 25 1 27 Done 29 5 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:44:57.34447164Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:44:57.344651949Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1782,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 280 280 280 F F F 1 020-039 158 158 158 280 280 280 F F F 2 040-059 155 155 155 274 274 274 F F F 3 060-079 159 159 159 281 281 281 F F F 4 080-099 158 158 158 280 280 280 F F F 5 100-119 144 144 144 257 257 257 F F F 6 120-139 150 150 150 266 266 266 F F F 7 140-159 159 159 159 281 281 281 F F F 8 160-179 157 157 157 279 279 279 F F F 9 180-199 160 149 160 282 263 282 F T F <--- 10 200-219 156 156 156 277 277 277 F F F 11 220-239 151 151 151 267 267 267 F F F 12 240-259 158 158 158 280 280 280 F F F 13 260-279 160 158 160 282 280 282 F F F <--- 14 280-299 160 156 160 282 278 282 F F F <--- 15 300-319 160 156 160 282 278 282 F T F <--- 16 320-339 159 159 159 281 281 281 F F F 17 340-359 150 150 150 265 265 265 F F F 18 360-379 160 152 160 282 268 282 F F F <--- 19 380-399 156 156 156 278 278 278 F F F 20 400-419 160 159 160 282 281 282 T T T <--- 21 420-439 159 159 159 281 281 281 F F F 22 440-459 158 158 158 280 280 280 F F F 23 460-479 160 156 160 282 278 282 F T F <--- 24 480-499 156 156 156 278 278 278 F F F 25 500-519 159 159 159 281 281 281 F F F 26 520-539 160 159 160 282 281 282 T F T <--- 27 540-559 160 157 160 282 279 282 T F T <--- 28 560-579 160 159 160 282 281 282 F F F <--- 29 580-599 156 156 156 278 278 278 F F F Max gen: 160, Max flush: 282 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 280 280 280 F F F 1 020-039 158 158 158 280 280 280 F F F 2 040-059 155 155 155 274 274 274 F F F 3 060-079 159 159 159 281 281 281 F F F 4 080-099 158 158 158 280 280 280 F F F 5 100-119 144 144 144 257 257 257 F F F 6 120-139 150 150 150 266 266 266 F F F 7 140-159 159 159 159 281 281 281 F F F 8 160-179 157 157 157 279 279 279 F F F 9 180-199 160 160 160 282 282 282 F F F 10 200-219 156 156 156 277 277 277 F F F 11 220-239 151 151 151 267 267 267 F F F 12 240-259 158 158 158 280 280 280 F F F 13 260-279 160 160 160 282 282 282 F F F 14 280-299 160 160 160 282 282 282 F F F 15 300-319 160 160 160 282 282 282 F F F 16 320-339 159 159 159 281 281 281 F F F 17 340-359 150 150 150 265 265 265 F F F 18 360-379 160 160 160 282 282 282 F F F 19 380-399 156 156 156 278 278 278 F F F 20 400-419 161 161 161 283 283 283 F F F 21 420-439 159 159 159 281 281 281 F F F 22 440-459 158 158 158 280 280 280 F F F 23 460-479 160 160 160 282 282 282 F F F 24 480-499 156 156 156 278 278 278 F F F 25 500-519 159 159 159 281 281 281 F F F 26 520-539 161 161 161 283 283 283 F F F 27 540-559 161 161 161 283 283 283 F F F 28 560-579 160 160 160 282 282 282 F F F 29 580-599 156 156 156 278 278 278 F F F Max gen: 161, Max flush: 283 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.590185393Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:01.597290463Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.597346187Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.604187518Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.604440639Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.604464376Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.604473035Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.604484594Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790} {"msg":"Crucible 03d789d5-9ab7-4982-ade9-7b317f3f6588 has session id: 02566585-9e47-4b83-9e2f-548b88660118","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.604827944Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"Upstairs opts: Upstairs UUID: 03d789d5-9ab7-4982-ade9-7b317f3f6588, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.60485803Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"Crucible stats registered with UUID: 03d789d5-9ab7-4982-ade9-7b317f3f6588","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.604870609Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:01.604879887Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.60503109Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.605059017Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.605070365Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.605227207Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.605251264Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790} {"msg":"03d789d5-9ab7-4982-ade9-7b317f3f6588 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.605268872Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.605277831Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"client":"0","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.60528759Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"client":"1","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.605297209Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"client":"2","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.605306928Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.605316786Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.605326875Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"ds_connection connected from Ok(127.0.0.1:46555)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.60537439Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"ds_connection connected from Ok(127.0.0.1:42163)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.605389388Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"ds_connection connected from Ok(127.0.0.1:45728)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.605402167Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606003046Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"client":"1","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606091396Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"client":"2","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606104065Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"client":"0","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606301542Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606347666Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"[0]R flush_numbers[0..12]: [280, 280, 274, 281, 280, 257, 266, 281, 279, 282, 277, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606411199Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"downstairs","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"[0]R generation[0..12]: [158, 158, 155, 159, 158, 144, 150, 159, 157, 160, 156, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606422677Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"downstairs","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606433606Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"downstairs","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"[1]R flush_numbers[0..12]: [280, 280, 274, 281, 280, 257, 266, 281, 279, 282, 277, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606444215Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"downstairs","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"[1]R generation[0..12]: [158, 158, 155, 159, 158, 144, 150, 159, 157, 160, 156, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606454694Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"downstairs","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606465612Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"downstairs","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"[2]R flush_numbers[0..12]: [280, 280, 274, 281, 280, 257, 266, 281, 279, 282, 277, 267]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606476231Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"downstairs","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"[2]R generation[0..12]: [158, 158, 155, 159, 158, 144, 150, 159, 157, 160, 156, 151]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.60648674Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"downstairs","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606500608Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"downstairs","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606510067Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"downstairs","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606519846Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"downstairs","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606529375Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"downstairs","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606538684Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"":"downstairs","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606548453Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606557062Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"client":"0","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606566711Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"client":"1","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606576459Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"client":"2","":"downstairs","session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606586268Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606595267Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"03d789d5-9ab7-4982-ade9-7b317f3f6588 is now active with session: 02566585-9e47-4b83-9e2f-548b88660118","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606605146Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606613875Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"session_id":"02566585-9e47-4b83-9e2f-548b88660118"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606622994Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790} {"msg":"Activated sub_volume 03d789d5-9ab7-4982-ade9-7b317f3f6588","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606630873Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606639012Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.606925309Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"task":"crutest"} 01/30 Write block 244 len 40960 data: 18 15 14 14 15 12 15 15 14 13 02/30 Write block 295 len 8192 data: 14 15 03/30 Read block 178 len 32768 04/30 Write block 501 len 20480 data: 12 10 13 13 15 05/30 Read block 361 len 16384 06/30 Flush 07/30 Write block 103 len 24576 data: 6 7 7 12 10 10 08/30 Read block 419 len 28672 09/30 Read block 276 len 36864 10/30 Write block 187 len 32768 data: 6 9 14 16 16 13 12 12 11/30 Write block 219 len 36864 data: 6 8 8 9 13 12 12 12 13 12/30 Read block 336 len 24576 13/30 Write block 267 len 32768 data: 15 17 20 19 17 16 17 16 14/30 Read block 290 len 4096 15/30 Read block 424 len 4096 16/30 Read block 403 len 20480 17/30 Write block 397 len 8192 data: 14 16 18/30 Read block 39 len 24576 19/30 Read block 257 len 16384 20/30 Write block 382 len 24576 data: 11 14 11 10 12 10 21/30 Write block 261 len 40960 data: 8 9 10 10 8 11 16 18 21 20 22/30 Write block 208 len 12288 data: 12 15 9 23/30 Write block 387 len 4096 data: 11 24/30 Read block 238 len 4096 25/30 Write block 8 len 36864 data: 16 13 15 14 14 13 10 9 10 26/30 Read block 537 len 24576 27/30 Write block 226 len 24576 data: 13 14 11 10 7 9 28/30 Write block 358 len 24576 data: 10 10 12 11 10 11 29/30 Write block 145 len 28672 data: 16 14 16 17 15 16 17 30/30 Read block 7 len 24576 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Sent Done Done false 1001 Acked Write 2 Sent Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Write 5 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Read 1 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Read 6 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Write 9 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Write 7 Sent Done Sent false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 2 32 Done 0 30 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1005)) Some(JobId(1005)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:01.615613055Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:01.615797883Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1790,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 280 280 280 F T T <--- 1 020-039 158 158 158 280 280 280 F F F 2 040-059 155 155 155 274 274 274 F F F 3 060-079 159 159 159 281 281 281 F F F 4 080-099 158 158 158 280 280 280 F F F 5 100-119 144 144 144 257 257 257 F T T <--- 6 120-139 150 150 150 266 266 266 F F F 7 140-159 159 159 159 281 281 281 F T T <--- 8 160-179 157 157 157 279 279 279 F F F 9 180-199 160 160 160 282 282 282 F T T <--- 10 200-219 156 156 156 277 277 277 F T T <--- 11 220-239 151 151 151 267 267 267 F T T <--- 12 240-259 158 162 162 280 284 284 F F F <--- 13 260-279 160 160 160 282 282 282 F T T <--- 14 280-299 160 162 162 282 284 284 F F F <--- 15 300-319 160 160 160 282 282 282 F F F 16 320-339 159 159 159 281 281 281 F F F 17 340-359 150 150 150 265 265 265 F T T <--- 18 360-379 160 160 160 282 282 282 F T T <--- 19 380-399 156 156 156 278 278 278 F T T <--- 20 400-419 161 161 161 283 283 283 F F F 21 420-439 159 159 159 281 281 281 F F F 22 440-459 158 158 158 280 280 280 F F F 23 460-479 160 160 160 282 282 282 F F F 24 480-499 156 156 156 278 278 278 F F F 25 500-519 159 162 162 281 284 284 F F F <--- 26 520-539 161 161 161 283 283 283 F F F 27 540-559 161 161 161 283 283 283 F F F 28 560-579 160 160 160 282 282 282 F F F 29 580-599 156 156 156 278 278 278 F F F Max gen: 162, Max flush: 284 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 285 285 285 F F F 1 020-039 158 158 158 280 280 280 F F F 2 040-059 155 155 155 274 274 274 F F F 3 060-079 159 159 159 281 281 281 F F F 4 080-099 158 158 158 280 280 280 F F F 5 100-119 163 163 163 285 285 285 F F F 6 120-139 150 150 150 266 266 266 F F F 7 140-159 163 163 163 285 285 285 F F F 8 160-179 157 157 157 279 279 279 F F F 9 180-199 163 163 163 285 285 285 F F F 10 200-219 163 163 163 285 285 285 F F F 11 220-239 163 163 163 285 285 285 F F F 12 240-259 162 162 162 284 284 284 F F F 13 260-279 163 163 163 285 285 285 F F F 14 280-299 162 162 162 284 284 284 F F F 15 300-319 160 160 160 282 282 282 F F F 16 320-339 159 159 159 281 281 281 F F F 17 340-359 163 163 163 285 285 285 F F F 18 360-379 163 163 163 285 285 285 F F F 19 380-399 163 163 163 285 285 285 F F F 20 400-419 161 161 161 283 283 283 F F F 21 420-439 159 159 159 281 281 281 F F F 22 440-459 158 158 158 280 280 280 F F F 23 460-479 160 160 160 282 282 282 F F F 24 480-499 156 156 156 278 278 278 F F F 25 500-519 162 162 162 284 284 284 F F F 26 520-539 161 161 161 283 283 283 F F F 27 540-559 161 161 161 283 283 283 F F F 28 560-579 160 160 160 282 282 282 F F F 29 580-599 156 156 156 278 278 278 F F F Max gen: 163, Max flush: 285 Begin loop 82 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.872784339Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.880940437Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881232183Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881262079Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881270898Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881282227Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798} {"msg":"Crucible 5880e9f3-ec37-4d7b-912a-ffe6b85299f7 has session id: 44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881621897Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"Upstairs opts: Upstairs UUID: 5880e9f3-ec37-4d7b-912a-ffe6b85299f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881640645Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"Crucible stats registered with UUID: 5880e9f3-ec37-4d7b-912a-ffe6b85299f7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881650464Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:05.881659613Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881796917Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881827103Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881840342Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881946299Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881957138Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798} {"msg":"5880e9f3-ec37-4d7b-912a-ffe6b85299f7 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881973816Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881982595Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"client":"0","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.881992344Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"client":"1","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.882002073Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"client":"2","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.882011752Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.882021691Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.882031709Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:50537)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.882092702Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:35783)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.882103711Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:33341)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.882121579Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.882753765Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"client":"2","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.882777192Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"client":"1","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.882858493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"client":"0","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883067259Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883084327Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"[0]R flush_numbers[0..12]: [285, 280, 274, 281, 280, 285, 266, 285, 279, 285, 285, 285]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.88313665Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"downstairs","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"[0]R generation[0..12]: [163, 158, 155, 159, 158, 163, 150, 163, 157, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883148229Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"downstairs","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883159198Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"downstairs","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"[1]R flush_numbers[0..12]: [285, 280, 274, 281, 280, 285, 266, 285, 279, 285, 285, 285]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883169877Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"downstairs","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"[1]R generation[0..12]: [163, 158, 155, 159, 158, 163, 150, 163, 157, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883180435Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"downstairs","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883191344Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"downstairs","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"[2]R flush_numbers[0..12]: [285, 280, 274, 281, 280, 285, 266, 285, 279, 285, 285, 285]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883201933Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"downstairs","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"[2]R generation[0..12]: [163, 158, 155, 159, 158, 163, 150, 163, 157, 163, 163, 163]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883212472Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"downstairs","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.88322332Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"downstairs","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883232699Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"downstairs","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883242638Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"downstairs","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883252177Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"downstairs","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883262996Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"":"downstairs","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883272705Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883281474Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"client":"0","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883291122Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"client":"1","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883306521Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"client":"2","":"downstairs","session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883316349Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883325518Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"5880e9f3-ec37-4d7b-912a-ffe6b85299f7 is now active with session: 44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883335467Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883344136Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"session_id":"44ea4b8b-7c33-4fcf-a1d1-51ebbabafbb3"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883353395Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798} {"msg":"Activated sub_volume 5880e9f3-ec37-4d7b-912a-ffe6b85299f7","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883361374Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883369453Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.883561271Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"task":"crutest"} 01/30 Write block 487 len 20480 data: 10 13 12 14 14 02/30 Read block 509 len 40960 03/30 Flush 04/30 Write block 99 len 20480 data: 8 8 7 7 7 05/30 Read block 101 len 36864 06/30 Write block 125 len 8192 data: 16 16 07/30 Read block 238 len 4096 08/30 Write block 453 len 36864 data: 5 6 6 7 9 10 9 8 7 09/30 Write block 222 len 32768 data: 10 14 13 13 14 15 12 11 10/30 Write block 560 len 4096 data: 11 11/30 Read block 237 len 8192 12/30 Read block 65 len 40960 13/30 Write block 293 len 4096 data: 13 14/30 Read block 581 len 20480 15/30 Flush 16/30 Write block 521 len 4096 data: 13 17/30 Write block 414 len 24576 data: 13 13 15 15 13 14 18/30 Read block 87 len 36864 19/30 Read block 144 len 16384 20/30 Read block 534 len 28672 21/30 Write block 231 len 16384 data: 10 10 10 10 22/30 Write block 587 len 16384 data: 10 9 12 11 23/30 Write block 532 len 28672 data: 13 15 14 14 13 10 13 24/30 Read block 371 len 20480 25/30 Write block 173 len 16384 data: 12 8 8 9 26/30 Read block 558 len 12288 27/30 Read block 512 len 36864 28/30 Read block 450 len 40960 29/30 Read block 129 len 40960 30/30 Read block 554 len 12288 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Read 10 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Write 2 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Write 1 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Read 5 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:05.892081176Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:05.892258336Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1798,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 285 285 285 F F F 1 020-039 158 158 158 280 280 280 F F F 2 040-059 155 155 155 274 274 274 F F F 3 060-079 159 159 159 281 281 281 F F F 4 080-099 164 158 164 287 280 287 F F F <--- 5 100-119 164 163 164 287 285 287 F F F <--- 6 120-139 164 150 164 287 266 287 F F F <--- 7 140-159 163 163 163 285 285 285 F F F 8 160-179 157 157 157 279 279 279 T F T <--- 9 180-199 163 163 163 285 285 285 F F F 10 200-219 163 163 163 285 285 285 F F F 11 220-239 164 163 164 287 285 287 T F T <--- 12 240-259 162 162 162 284 284 284 F F F 13 260-279 163 163 163 285 285 285 F F F 14 280-299 164 162 164 287 284 287 F F F <--- 15 300-319 160 160 160 282 282 282 F F F 16 320-339 159 159 159 281 281 281 F F F 17 340-359 163 163 163 285 285 285 F F F 18 360-379 163 163 163 285 285 285 F F F 19 380-399 163 163 163 285 285 285 F F F 20 400-419 161 161 161 283 283 283 T F T <--- 21 420-439 159 159 159 281 281 281 F F F 22 440-459 164 158 164 287 280 287 F F F <--- 23 460-479 164 160 164 287 282 287 F F F <--- 24 480-499 164 156 164 286 278 286 F F F <--- 25 500-519 162 162 162 284 284 284 F F F 26 520-539 161 161 161 283 283 283 T F T <--- 27 540-559 161 161 161 283 283 283 F F F 28 560-579 164 160 164 287 282 287 F F F <--- 29 580-599 156 156 156 278 278 278 T F T <--- Max gen: 164, Max flush: 287 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 285 285 285 F F F 1 020-039 158 158 158 280 280 280 F F F 2 040-059 155 155 155 274 274 274 F F F 3 060-079 159 159 159 281 281 281 F F F 4 080-099 164 164 164 287 287 287 F F F 5 100-119 164 164 164 287 287 287 F F F 6 120-139 164 164 164 287 287 287 F F F 7 140-159 163 163 163 285 285 285 F F F 8 160-179 165 165 165 288 288 288 F F F 9 180-199 163 163 163 285 285 285 F F F 10 200-219 163 163 163 285 285 285 F F F 11 220-239 165 165 165 288 288 288 F F F 12 240-259 162 162 162 284 284 284 F F F 13 260-279 163 163 163 285 285 285 F F F 14 280-299 164 164 164 287 287 287 F F F 15 300-319 160 160 160 282 282 282 F F F 16 320-339 159 159 159 281 281 281 F F F 17 340-359 163 163 163 285 285 285 F F F 18 360-379 163 163 163 285 285 285 F F F 19 380-399 163 163 163 285 285 285 F F F 20 400-419 165 165 165 288 288 288 F F F 21 420-439 159 159 159 281 281 281 F F F 22 440-459 164 164 164 287 287 287 F F F 23 460-479 164 164 164 287 287 287 F F F 24 480-499 164 164 164 286 286 286 F F F 25 500-519 162 162 162 284 284 284 F F F 26 520-539 165 165 165 288 288 288 F F F 27 540-559 161 161 161 283 283 283 F F F 28 560-579 164 164 164 287 287 287 F F F 29 580-599 165 165 165 288 288 288 F F F Max gen: 165, Max flush: 288 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.139437427Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.147613112Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.14788853Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.147913357Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.147925456Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.147937105Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806} {"msg":"Crucible cf7e701b-bb13-462e-8692-fa7f25db0fe8 has session id: 74b0722e-e6ee-4702-8e80-6f09c3eeae0b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148225341Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"Upstairs opts: Upstairs UUID: cf7e701b-bb13-462e-8692-fa7f25db0fe8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148245229Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"Crucible stats registered with UUID: cf7e701b-bb13-462e-8692-fa7f25db0fe8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148255368Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:10.148264566Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.14840585Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148420378Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148430337Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148556532Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148572091Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806} {"msg":"cf7e701b-bb13-462e-8692-fa7f25db0fe8 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148610526Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148620005Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"client":"0","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148629674Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"client":"1","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148641202Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"client":"2","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148650781Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.14866079Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148682218Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61879)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148708415Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44001)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148719943Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50770)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.148730102Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149303505Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"client":"1","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149329732Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"client":"0","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149454778Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"client":"2","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149588802Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.14960586Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"[0]R flush_numbers[0..12]: [285, 280, 274, 281, 287, 287, 287, 285, 288, 285, 285, 288]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149716727Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"downstairs","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"[0]R generation[0..12]: [163, 158, 155, 159, 164, 164, 164, 163, 165, 163, 163, 165]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149728226Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"downstairs","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149739294Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"downstairs","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"[1]R flush_numbers[0..12]: [285, 280, 274, 281, 287, 287, 287, 285, 288, 285, 285, 288]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149752263Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"downstairs","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"[1]R generation[0..12]: [163, 158, 155, 159, 164, 164, 164, 163, 165, 163, 163, 165]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149762822Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"downstairs","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.14977357Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"downstairs","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"[2]R flush_numbers[0..12]: [285, 280, 274, 281, 287, 287, 287, 285, 288, 285, 285, 288]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149784169Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"downstairs","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"[2]R generation[0..12]: [163, 158, 155, 159, 164, 164, 164, 163, 165, 163, 163, 165]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149794588Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"downstairs","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149805557Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"downstairs","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149815055Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"downstairs","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149824774Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"downstairs","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149834273Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"downstairs","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149843722Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"":"downstairs","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149853701Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.14986267Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"client":"0","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149872269Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"client":"1","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149881908Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"client":"2","":"downstairs","session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149891627Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149901895Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"cf7e701b-bb13-462e-8692-fa7f25db0fe8 is now active with session: 74b0722e-e6ee-4702-8e80-6f09c3eeae0b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149911814Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149920553Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"session_id":"74b0722e-e6ee-4702-8e80-6f09c3eeae0b"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149929762Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806} {"msg":"Activated sub_volume cf7e701b-bb13-462e-8692-fa7f25db0fe8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.149937781Z","hostname"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.14995128Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.150219098Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"task":"crutest"} 01/30 Write block 302 len 8192 data: 19 19 02/30 Read block 164 len 36864 03/30 Write block 202 len 32768 data: 12 13 13 12 11 9 13 16 04/30 Read block 495 len 28672 05/30 Flush 06/30 Write block 456 len 20480 data: 8 10 11 10 9 07/30 Read block 351 len 8192 08/30 Flush 09/30 Write block 540 len 12288 data: 11 12 13 10/30 Read block 483 len 20480 11/30 Write block 535 len 24576 data: 15 14 11 14 13 12 12/30 Read block 375 len 8192 13/30 Read block 46 len 24576 14/30 Write block 353 len 28672 data: 18 19 15 12 9 11 11 15/30 Read block 332 len 28672 16/30 Flush 17/30 Write block 287 len 12288 data: 12 11 13 18/30 Write block 76 len 8192 data: 12 10 19/30 Read block 442 len 32768 20/30 Read block 221 len 36864 21/30 Write block 530 len 12288 data: 13 13 14 22/30 Write block 567 len 24576 data: 13 15 10 12 14 14 23/30 Write block 156 len 20480 data: 14 13 12 16 14 24/30 Write block 221 len 40960 data: 9 11 15 14 14 15 16 13 12 8 25/30 Write block 292 len 28672 data: 8 14 15 15 16 16 17 26/30 Flush 27/30 Write block 385 len 36864 data: 11 13 12 9 12 11 13 12 13 28/30 Read block 506 len 32768 29/30 Read block 458 len 12288 30/30 Read block 516 len 36864 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 8 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Write 10 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 28 29 Done 30 29 2 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:10.159064396Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:10.159248074Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1806,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 285 285 285 F F F 1 020-039 158 158 158 280 280 280 F F F 2 040-059 155 155 155 274 274 274 F F F 3 060-079 166 166 159 292 292 281 F F F <--- 4 080-099 164 164 164 287 287 287 F F F 5 100-119 164 164 164 287 287 287 F F F 6 120-139 164 164 164 287 287 287 F F F 7 140-159 166 166 163 292 292 285 F F F <--- 8 160-179 166 166 165 292 292 288 F F F <--- 9 180-199 163 163 163 285 285 285 F F F 10 200-219 166 166 163 289 289 285 F F F <--- 11 220-239 166 166 165 292 292 288 F F F <--- 12 240-259 162 162 162 284 284 284 F F F 13 260-279 163 163 163 285 285 285 F F F 14 280-299 166 166 164 292 292 287 F F F <--- 15 300-319 166 166 160 289 289 282 F F T <--- 16 320-339 159 159 159 281 281 281 F F F 17 340-359 166 166 163 291 291 285 F F F <--- 18 360-379 163 163 163 285 285 285 F F F 19 380-399 163 163 163 285 285 285 T T F <--- 20 400-419 165 165 165 288 288 288 F F F 21 420-439 159 159 159 281 281 281 F F F 22 440-459 166 166 164 290 290 287 F F F <--- 23 460-479 166 166 164 290 290 287 F F F <--- 24 480-499 164 164 164 286 286 286 F F F 25 500-519 162 162 162 284 284 284 F F F 26 520-539 166 166 165 292 292 288 F F F <--- 27 540-559 166 166 161 291 291 283 F F F <--- 28 560-579 166 166 164 292 292 287 F F F <--- 29 580-599 165 165 165 288 288 288 F F F Max gen: 166, Max flush: 292 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 285 285 285 F F F 1 020-039 158 158 158 280 280 280 F F F 2 040-059 155 155 155 274 274 274 F F F 3 060-079 166 166 166 292 292 292 F F F 4 080-099 164 164 164 287 287 287 F F F 5 100-119 164 164 164 287 287 287 F F F 6 120-139 164 164 164 287 287 287 F F F 7 140-159 166 166 166 292 292 292 F F F 8 160-179 166 166 166 292 292 292 F F F 9 180-199 163 163 163 285 285 285 F F F 10 200-219 166 166 166 289 289 289 F F F 11 220-239 166 166 166 292 292 292 F F F 12 240-259 162 162 162 284 284 284 F F F 13 260-279 163 163 163 285 285 285 F F F 14 280-299 166 166 166 292 292 292 F F F 15 300-319 166 166 166 289 289 289 F F F 16 320-339 159 159 159 281 281 281 F F F 17 340-359 166 166 166 291 291 291 F F F 18 360-379 163 163 163 285 285 285 F F F 19 380-399 167 167 167 293 293 293 F F F 20 400-419 165 165 165 288 288 288 F F F 21 420-439 159 159 159 281 281 281 F F F 22 440-459 166 166 166 290 290 290 F F F 23 460-479 166 166 166 290 290 290 F F F 24 480-499 164 164 164 286 286 286 F F F 25 500-519 162 162 162 284 284 284 F F F 26 520-539 166 166 166 292 292 292 F F F 27 540-559 166 166 166 291 291 291 F F F 28 560-579 166 166 166 292 292 292 F F F 29 580-599 165 165 165 288 288 288 F F F Max gen: 167, Max flush: 293 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.408560956Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.416759419Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417069493Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.41709578Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417104699Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417116378Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814} {"msg":"Crucible fcd88f2d-c57e-4e4d-8ac7-e7758aca358b has session id: c3af7aaf-e4d5-4203-a12e-24921cdeafe5","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417446109Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"Upstairs opts: Upstairs UUID: fcd88f2d-c57e-4e4d-8ac7-e7758aca358b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417465917Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"Crucible stats registered with UUID: fcd88f2d-c57e-4e4d-8ac7-e7758aca358b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417477886Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:14.417486825Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417637057Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417651545Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417661744Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417822385Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417839283Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814} {"msg":"fcd88f2d-c57e-4e4d-8ac7-e7758aca358b active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417873689Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417883188Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"client":"0","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417892727Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"client":"1","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417902306Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"client":"2","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417911715Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417921684Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.417931543Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"ds_connection connected from Ok(127.0.0.1:47101)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.418003554Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"ds_connection connected from Ok(127.0.0.1:48527)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.418109552Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"ds_connection connected from Ok(127.0.0.1:54894)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.418124Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.418620752Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"client":"0","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.418674986Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"client":"1","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.418686565Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"client":"2","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.418972561Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419038513Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"[0]R flush_numbers[0..12]: [285, 280, 274, 292, 287, 287, 287, 292, 292, 285, 289, 292]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419058451Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"downstairs","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"[0]R generation[0..12]: [163, 158, 155, 166, 164, 164, 164, 166, 166, 163, 166, 166]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.41906913Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"downstairs","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419080279Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"downstairs","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"[1]R flush_numbers[0..12]: [285, 280, 274, 292, 287, 287, 287, 292, 292, 285, 289, 292]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419090747Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"downstairs","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"[1]R generation[0..12]: [163, 158, 155, 166, 164, 164, 164, 166, 166, 163, 166, 166]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419101466Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"downstairs","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419112365Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"downstairs","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"[2]R flush_numbers[0..12]: [285, 280, 274, 292, 287, 287, 287, 292, 292, 285, 289, 292]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419123234Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"downstairs","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"[2]R generation[0..12]: [163, 158, 155, 166, 164, 164, 164, 166, 166, 163, 166, 166]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419135922Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"downstairs","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419146881Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"downstairs","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.41915617Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"downstairs","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419165769Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"downstairs","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419174958Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"downstairs","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419184316Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"":"downstairs","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419194365Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419203304Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"client":"0","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419213253Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"client":"1","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419223052Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"client":"2","":"downstairs","session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419233131Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.41924217Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"fcd88f2d-c57e-4e4d-8ac7-e7758aca358b is now active with session: c3af7aaf-e4d5-4203-a12e-24921cdeafe5","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419252069Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419260768Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"session_id":"c3af7aaf-e4d5-4203-a12e-24921cdeafe5"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419269876Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814} {"msg":"Activated sub_volume fcd88f2d-c57e-4e4d-8ac7-e7758aca358b","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419277945Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419285895Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.419644363Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"task":"crutest"} 01/30 Write block 59 len 16384 data: 14 14 15 14 02/30 Read block 323 len 36864 03/30 Read block 241 len 12288 04/30 Read block 170 len 32768 05/30 Write block 578 len 40960 data: 11 14 11 13 13 11 9 11 10 11 06/30 Flush 07/30 Write block 593 len 20480 data: 11 9 11 10 10 08/30 Flush 09/30 Write block 214 len 28672 data: 5 6 6 7 8 7 9 10/30 Flush 11/30 Write block 105 len 36864 data: 8 13 11 11 7 8 9 8 7 12/30 Read block 592 len 16384 13/30 Write block 529 len 12288 data: 10 14 14 14/30 Write block 433 len 36864 data: 15 16 14 13 11 12 15 14 18 15/30 Read block 272 len 12288 16/30 Read block 250 len 28672 17/30 Read block 219 len 24576 18/30 Write block 322 len 20480 data: 14 11 11 12 11 19/30 Read block 522 len 8192 20/30 Write block 382 len 12288 data: 12 15 12 21/30 Read block 309 len 4096 22/30 Write block 346 len 12288 data: 9 9 10 23/30 Read block 132 len 24576 24/30 Write block 165 len 12288 data: 12 13 11 25/30 Read block 212 len 36864 26/30 Read block 432 len 4096 27/30 Read block 509 len 12288 28/30 Write block 565 len 4096 data: 13 29/30 Read block 66 len 36864 30/30 Read block 165 len 28672 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Sent Done false 1001 Acked Read 9 Done Sent Done false 1002 Acked Read 3 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Write 10 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 9 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) None Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:14.428878005Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:14.429071622Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1814,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 285 285 285 F F F 1 020-039 158 158 158 280 280 280 F F F 2 040-059 168 155 168 294 274 294 F F F <--- 3 060-079 168 166 168 294 292 294 F F F <--- 4 080-099 164 164 164 287 287 287 F F F 5 100-119 164 164 164 287 287 287 T F T <--- 6 120-139 164 164 164 287 287 287 F F F 7 140-159 166 166 166 292 292 292 F F F 8 160-179 166 166 166 292 292 292 T F T <--- 9 180-199 163 163 163 285 285 285 F F F 10 200-219 168 166 168 296 289 296 F F F <--- 11 220-239 168 166 168 296 292 296 F F F <--- 12 240-259 162 162 162 284 284 284 F F F 13 260-279 163 163 163 285 285 285 F F F 14 280-299 166 166 166 292 292 292 F F F 15 300-319 166 166 166 289 289 289 F F F 16 320-339 159 159 159 281 281 281 T F T <--- 17 340-359 166 166 166 291 291 291 T F T <--- 18 360-379 163 163 163 285 285 285 F F F 19 380-399 167 167 167 293 293 293 T F T <--- 20 400-419 165 165 165 288 288 288 F F F 21 420-439 159 159 159 281 281 281 T F T <--- 22 440-459 166 166 166 290 290 290 T F T <--- 23 460-479 166 166 166 290 290 290 F F F 24 480-499 164 164 164 286 286 286 F F F 25 500-519 162 162 162 284 284 284 F F F 26 520-539 166 166 166 292 292 292 T F T <--- 27 540-559 166 166 166 291 291 291 F F F 28 560-579 168 166 168 294 292 294 T F T <--- 29 580-599 168 165 168 295 288 295 F F F <--- Max gen: 168, Max flush: 296 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 285 285 285 F F F 1 020-039 158 158 158 280 280 280 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 168 168 168 294 294 294 F F F 4 080-099 164 164 164 287 287 287 F F F 5 100-119 169 169 169 297 297 297 F F F 6 120-139 164 164 164 287 287 287 F F F 7 140-159 166 166 166 292 292 292 F F F 8 160-179 169 169 169 297 297 297 F F F 9 180-199 163 163 163 285 285 285 F F F 10 200-219 168 168 168 296 296 296 F F F 11 220-239 168 168 168 296 296 296 F F F 12 240-259 162 162 162 284 284 284 F F F 13 260-279 163 163 163 285 285 285 F F F 14 280-299 166 166 166 292 292 292 F F F 15 300-319 166 166 166 289 289 289 F F F 16 320-339 169 169 169 297 297 297 F F F 17 340-359 169 169 169 297 297 297 F F F 18 360-379 163 163 163 285 285 285 F F F 19 380-399 169 169 169 297 297 297 F F F 20 400-419 165 165 165 288 288 288 F F F 21 420-439 169 169 169 297 297 297 F F F 22 440-459 169 169 169 297 297 297 F F F 23 460-479 166 166 166 290 290 290 F F F 24 480-499 164 164 164 286 286 286 F F F 25 500-519 162 162 162 284 284 284 F F F 26 520-539 169 169 169 297 297 297 F F F 27 540-559 166 166 166 291 291 291 F F F 28 560-579 169 169 169 297 297 297 F F F 29 580-599 168 168 168 295 295 295 F F F Max gen: 169, Max flush: 297 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.681918612Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:18.689173535Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.68921621Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.696276806Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.696526637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.696556293Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.696564972Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.696576461Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822} {"msg":"Crucible bd567ee7-8860-442c-bd6f-d9dfcb2f5b99 has session id: 111ae154-9130-4749-9703-1baa5eead82a","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.696905402Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"Upstairs opts: Upstairs UUID: bd567ee7-8860-442c-bd6f-d9dfcb2f5b99, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.69692498Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"Crucible stats registered with UUID: bd567ee7-8860-442c-bd6f-d9dfcb2f5b99","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.696935299Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:18.696944458Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.69710169Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.697135566Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.697147074Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.697260851Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.697275139Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822} {"msg":"bd567ee7-8860-442c-bd6f-d9dfcb2f5b99 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.697298817Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.697308255Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"client":"0","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.697317894Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"client":"1","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.697327463Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"client":"2","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.697337052Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.697347131Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.69735698Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"ds_connection connected from Ok(127.0.0.1:53740)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.697395765Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"ds_connection connected from Ok(127.0.0.1:64664)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.697406794Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"ds_connection connected from Ok(127.0.0.1:61116)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.697416893Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698065627Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"client":"2","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698086795Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"client":"1","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698167325Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"client":"0","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698368102Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.69838581Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"[0]R flush_numbers[0..12]: [285, 280, 294, 294, 287, 297, 287, 292, 297, 285, 296, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698439443Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"downstairs","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"[0]R generation[0..12]: [163, 158, 168, 168, 164, 169, 164, 166, 169, 163, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698450842Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"downstairs","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698462071Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"downstairs","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"[1]R flush_numbers[0..12]: [285, 280, 294, 294, 287, 297, 287, 292, 297, 285, 296, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.69847271Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"downstairs","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"[1]R generation[0..12]: [163, 158, 168, 168, 164, 169, 164, 166, 169, 163, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698483408Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"downstairs","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698494407Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"downstairs","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"[2]R flush_numbers[0..12]: [285, 280, 294, 294, 287, 297, 287, 292, 297, 285, 296, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698505056Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"downstairs","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"[2]R generation[0..12]: [163, 158, 168, 168, 164, 169, 164, 166, 169, 163, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698515764Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"downstairs","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698526783Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"downstairs","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698536272Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"downstairs","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698546141Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"downstairs","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698557Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"downstairs","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698566679Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"":"downstairs","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698576507Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698585236Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"client":"0","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698594795Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"client":"1","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698604384Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"client":"2","":"downstairs","session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698614103Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698623382Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"bd567ee7-8860-442c-bd6f-d9dfcb2f5b99 is now active with session: 111ae154-9130-4749-9703-1baa5eead82a","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698633331Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.6986422Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"session_id":"111ae154-9130-4749-9703-1baa5eead82a"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698651649Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822} {"msg":"Activated sub_volume bd567ee7-8860-442c-bd6f-d9dfcb2f5b99","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698659678Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698667887Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.698944764Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"task":"crutest"} 01/30 Write block 550 len 20480 data: 22 21 21 21 20 02/30 Read block 202 len 24576 03/30 Read block 531 len 24576 04/30 Read block 377 len 20480 05/30 Write block 573 len 36864 data: 14 15 15 14 15 12 15 12 14 06/30 Read block 126 len 32768 07/30 Read block 121 len 8192 08/30 Read block 13 len 36864 09/30 Read block 366 len 12288 10/30 Read block 581 len 40960 11/30 Write block 483 len 8192 data: 10 12 12/30 Read block 456 len 8192 13/30 Read block 3 len 4096 14/30 Flush 15/30 Write block 423 len 32768 data: 9 7 5 10 11 12 12 12 16/30 Read block 26 len 12288 17/30 Read block 535 len 4096 18/30 Write block 81 len 36864 data: 9 12 15 14 14 16 14 15 19 19/30 Read block 57 len 36864 20/30 Read block 503 len 16384 21/30 Read block 293 len 8192 22/30 Read block 42 len 16384 23/30 Write block 256 len 32768 data: 12 13 7 7 6 9 10 11 24/30 Write block 135 len 36864 data: 12 10 9 10 11 12 11 13 14 25/30 Write block 587 len 28672 data: 12 10 13 12 9 10 12 26/30 Read block 101 len 40960 27/30 Write block 533 len 24576 data: 16 15 16 15 12 15 28/30 Write block 389 len 16384 data: 13 12 14 13 29/30 Write block 357 len 28672 data: 10 12 12 13 12 11 12 30/30 Read block 175 len 32768 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 9 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 2 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Write 8 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 21 0 0 21 Done 9 30 30 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:18.708380163Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:18.708553863Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1822,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 285 285 285 F F F 1 020-039 158 158 158 280 280 280 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 168 168 168 294 294 294 F F F 4 080-099 164 164 164 287 287 287 F T T <--- 5 100-119 169 169 169 297 297 297 F F F 6 120-139 164 164 164 287 287 287 F T T <--- 7 140-159 166 166 166 292 292 292 F T T <--- 8 160-179 169 169 169 297 297 297 F F F 9 180-199 163 163 163 285 285 285 F F F 10 200-219 168 168 168 296 296 296 F F F 11 220-239 168 168 168 296 296 296 F F F 12 240-259 162 162 162 284 284 284 F T T <--- 13 260-279 163 163 163 285 285 285 F T T <--- 14 280-299 166 166 166 292 292 292 F F F 15 300-319 166 166 166 289 289 289 F F F 16 320-339 169 169 169 297 297 297 F F F 17 340-359 169 169 169 297 297 297 F T T <--- 18 360-379 163 163 163 285 285 285 F T T <--- 19 380-399 169 169 169 297 297 297 F T T <--- 20 400-419 165 165 165 288 288 288 F F F 21 420-439 169 169 169 297 297 297 F T T <--- 22 440-459 169 169 169 297 297 297 F F F 23 460-479 166 166 166 290 290 290 F F F 24 480-499 164 170 170 286 298 298 F F F <--- 25 500-519 162 162 162 284 284 284 F F F 26 520-539 169 169 169 297 297 297 F T T <--- 27 540-559 166 170 170 291 298 298 T F F <--- 28 560-579 169 170 170 297 298 298 T F F <--- 29 580-599 168 170 170 295 298 298 T T T <--- Max gen: 170, Max flush: 298 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 285 285 285 F F F 1 020-039 158 158 158 280 280 280 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 168 168 168 294 294 294 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 169 169 169 297 297 297 F F F 6 120-139 171 171 171 299 299 299 F F F 7 140-159 171 171 171 299 299 299 F F F 8 160-179 169 169 169 297 297 297 F F F 9 180-199 163 163 163 285 285 285 F F F 10 200-219 168 168 168 296 296 296 F F F 11 220-239 168 168 168 296 296 296 F F F 12 240-259 171 171 171 299 299 299 F F F 13 260-279 171 171 171 299 299 299 F F F 14 280-299 166 166 166 292 292 292 F F F 15 300-319 166 166 166 289 289 289 F F F 16 320-339 169 169 169 297 297 297 F F F 17 340-359 171 171 171 299 299 299 F F F 18 360-379 171 171 171 299 299 299 F F F 19 380-399 171 171 171 299 299 299 F F F 20 400-419 165 165 165 288 288 288 F F F 21 420-439 171 171 171 299 299 299 F F F 22 440-459 169 169 169 297 297 297 F F F 23 460-479 166 166 166 290 290 290 F F F 24 480-499 170 170 170 298 298 298 F F F 25 500-519 162 162 162 284 284 284 F F F 26 520-539 171 171 171 299 299 299 F F F 27 540-559 170 170 170 298 298 298 F F F 28 560-579 170 170 170 298 298 298 F F F 29 580-599 171 171 171 299 299 299 F F F Max gen: 171, Max flush: 299 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.977727576Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.986025077Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.986304635Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.986332142Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.986341091Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.986352709Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830} {"msg":"Crucible 2eaa629a-ab5e-46c2-b5ac-2423bc43a9b2 has session id: 8d550238-fff1-41d4-8d11-d342126a51ec","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.986713427Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"Upstairs opts: Upstairs UUID: 2eaa629a-ab5e-46c2-b5ac-2423bc43a9b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.986741704Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"Crucible stats registered with UUID: 2eaa629a-ab5e-46c2-b5ac-2423bc43a9b2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.986752483Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:22.986761551Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.986904895Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.986932561Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.98694335Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.987070935Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.987089933Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830} {"msg":"2eaa629a-ab5e-46c2-b5ac-2423bc43a9b2 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.98711893Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.987128439Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"client":"0","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.987140327Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"client":"1","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.987150246Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"client":"2","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.987160025Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.987169924Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.987179863Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:55325)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.987254244Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:53523)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.987279011Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:61623)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.987346723Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.987945083Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"client":"2","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988017495Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"client":"1","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988039402Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"client":"0","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988252587Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988272035Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"[0]R flush_numbers[0..12]: [285, 280, 294, 294, 299, 297, 299, 299, 297, 285, 296, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988325839Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"downstairs","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"[0]R generation[0..12]: [163, 158, 168, 168, 171, 169, 171, 171, 169, 163, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988337358Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"downstairs","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988350546Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"downstairs","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"[1]R flush_numbers[0..12]: [285, 280, 294, 294, 299, 297, 299, 299, 297, 285, 296, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988361175Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"downstairs","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"[1]R generation[0..12]: [163, 158, 168, 168, 171, 169, 171, 171, 169, 163, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988371633Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"downstairs","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988382522Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"downstairs","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"[2]R flush_numbers[0..12]: [285, 280, 294, 294, 299, 297, 299, 299, 297, 285, 296, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988393051Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"downstairs","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"[2]R generation[0..12]: [163, 158, 168, 168, 171, 169, 171, 171, 169, 163, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.98840349Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"downstairs","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988414399Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"downstairs","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988423917Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"downstairs","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988433606Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"downstairs","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"Next flush: 300","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988442835Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"downstairs","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988452194Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"":"downstairs","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988461963Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988470702Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"client":"0","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988480441Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"client":"1","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.98849125Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"client":"2","":"downstairs","session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988501118Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988510117Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"2eaa629a-ab5e-46c2-b5ac-2423bc43a9b2 is now active with session: 8d550238-fff1-41d4-8d11-d342126a51ec","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988520026Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988528825Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"session_id":"8d550238-fff1-41d4-8d11-d342126a51ec"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988537884Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830} {"msg":"Activated sub_volume 2eaa629a-ab5e-46c2-b5ac-2423bc43a9b2","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988545833Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988554072Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.988859237Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"task":"crutest"} 01/30 Write block 274 len 28672 data: 17 17 16 14 15 16 13 02/30 Write block 555 len 4096 data: 22 03/30 Read block 482 len 24576 04/30 Read block 577 len 32768 05/30 Write block 411 len 16384 data: 14 14 14 14 06/30 Flush 07/30 Write block 27 len 24576 data: 18 21 20 21 22 20 08/30 Write block 31 len 8192 data: 23 21 09/30 Read block 294 len 4096 10/30 Read block 253 len 24576 11/30 Read block 13 len 12288 12/30 Read block 359 len 16384 13/30 Flush 14/30 Write block 550 len 24576 data: 23 22 22 22 21 23 15/30 Read block 290 len 12288 16/30 Write block 246 len 24576 data: 15 15 16 13 16 16 17/30 Read block 44 len 20480 18/30 Write block 305 len 20480 data: 20 18 19 16 16 19/30 Read block 387 len 36864 20/30 Read block 266 len 24576 21/30 Read block 189 len 12288 22/30 Write block 1 len 28672 data: 8 11 11 15 16 17 17 23/30 Write block 434 len 28672 data: 17 15 14 12 13 16 15 24/30 Write block 450 len 20480 data: 12 8 7 6 7 25/30 Write block 329 len 28672 data: 12 14 12 14 16 15 16 26/30 Read block 16 len 40960 27/30 Read block 188 len 36864 28/30 Read block 595 len 12288 29/30 Write block 72 len 28672 data: 12 12 13 13 13 11 13 30/30 Read block 450 len 8192 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 6 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 6 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) None Some(JobId(1012)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:22.997831229Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:22.998012698Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1830,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 285 285 285 T F T <--- 1 020-039 172 158 172 301 280 301 F F F <--- 2 040-059 168 168 168 294 294 294 F F F 3 060-079 168 168 168 294 294 294 T F T <--- 4 080-099 171 171 171 299 299 299 F F F 5 100-119 169 169 169 297 297 297 F F F 6 120-139 171 171 171 299 299 299 F F F 7 140-159 171 171 171 299 299 299 F F F 8 160-179 169 169 169 297 297 297 F F F 9 180-199 163 163 163 285 285 285 F F F 10 200-219 168 168 168 296 296 296 F F F 11 220-239 168 168 168 296 296 296 F F F 12 240-259 171 171 171 299 299 299 T F T <--- 13 260-279 172 171 172 300 299 300 F T F <--- 14 280-299 172 166 172 300 292 300 F T F <--- 15 300-319 166 166 166 289 289 289 T F T <--- 16 320-339 169 169 169 297 297 297 T F T <--- 17 340-359 171 171 171 299 299 299 F F F 18 360-379 171 171 171 299 299 299 F F F 19 380-399 171 171 171 299 299 299 F F F 20 400-419 172 165 172 300 288 300 F F F <--- 21 420-439 171 171 171 299 299 299 T F T <--- 22 440-459 169 169 169 297 297 297 T F T <--- 23 460-479 166 166 166 290 290 290 F F F 24 480-499 170 170 170 298 298 298 F F F 25 500-519 162 162 162 284 284 284 F F F 26 520-539 171 171 171 299 299 299 F F F 27 540-559 172 170 172 300 298 300 T T T <--- 28 560-579 170 170 170 298 298 298 F F F 29 580-599 171 171 171 299 299 299 F F F Max gen: 172, Max flush: 301 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 302 302 302 F F F 1 020-039 172 172 172 301 301 301 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 173 173 173 302 302 302 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 169 169 169 297 297 297 F F F 6 120-139 171 171 171 299 299 299 F F F 7 140-159 171 171 171 299 299 299 F F F 8 160-179 169 169 169 297 297 297 F F F 9 180-199 163 163 163 285 285 285 F F F 10 200-219 168 168 168 296 296 296 F F F 11 220-239 168 168 168 296 296 296 F F F 12 240-259 173 173 173 302 302 302 F F F 13 260-279 172 172 172 300 300 300 F F F 14 280-299 172 172 172 300 300 300 F F F 15 300-319 173 173 173 302 302 302 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 171 171 171 299 299 299 F F F 18 360-379 171 171 171 299 299 299 F F F 19 380-399 171 171 171 299 299 299 F F F 20 400-419 172 172 172 300 300 300 F F F 21 420-439 173 173 173 302 302 302 F F F 22 440-459 173 173 173 302 302 302 F F F 23 460-479 166 166 166 290 290 290 F F F 24 480-499 170 170 170 298 298 298 F F F 25 500-519 162 162 162 284 284 284 F F F 26 520-539 171 171 171 299 299 299 F F F 27 540-559 173 173 173 302 302 302 F F F 28 560-579 170 170 170 298 298 298 F F F 29 580-599 171 171 171 299 299 299 F F F Max gen: 173, Max flush: 302 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.248435251Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:27.255647989Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.255698543Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.262547923Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.262800394Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.262826251Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.26283469Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.262848868Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838} {"msg":"Crucible 0ac2ce2e-de0b-415a-82d4-2fd0354a0d15 has session id: 749c37b8-d8e2-44f2-ace9-7d69d329d0b3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263193098Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"Upstairs opts: Upstairs UUID: 0ac2ce2e-de0b-415a-82d4-2fd0354a0d15, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263234703Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"Crucible stats registered with UUID: 0ac2ce2e-de0b-415a-82d4-2fd0354a0d15","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263245272Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:27.263254391Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263389725Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263419392Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.26343039Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263573304Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263588192Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838} {"msg":"0ac2ce2e-de0b-415a-82d4-2fd0354a0d15 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263623128Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263632637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"client":"0","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263642476Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"client":"1","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263652204Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"client":"2","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263661773Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263673732Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263716807Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:33161)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263766331Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:33123)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.26385953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"ds_connection connected from Ok(127.0.0.1:51445)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.263871939Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.264496126Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"client":"2","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.264519083Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"client":"1","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.264602533Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"client":"0","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.264849075Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.264871512Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"[0]R flush_numbers[0..12]: [302, 301, 294, 302, 299, 297, 299, 299, 297, 285, 296, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.264937504Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"downstairs","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"[0]R generation[0..12]: [173, 172, 168, 173, 171, 169, 171, 171, 169, 163, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.264949293Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"downstairs","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.264960262Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"downstairs","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"[1]R flush_numbers[0..12]: [302, 301, 294, 302, 299, 297, 299, 299, 297, 285, 296, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.26497117Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"downstairs","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"[1]R generation[0..12]: [173, 172, 168, 173, 171, 169, 171, 171, 169, 163, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.264981979Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"downstairs","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.264995518Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"downstairs","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"[2]R flush_numbers[0..12]: [302, 301, 294, 302, 299, 297, 299, 299, 297, 285, 296, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265006556Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"downstairs","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"[2]R generation[0..12]: [173, 172, 168, 173, 171, 169, 171, 171, 169, 163, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265017225Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"downstairs","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265028144Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"downstairs","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265037613Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"downstairs","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265047442Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"downstairs","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"Next flush: 303","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.26505667Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"downstairs","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265066079Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"":"downstairs","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265075798Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265084567Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"client":"0","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265094116Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"client":"1","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265103985Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"client":"2","":"downstairs","session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265113714Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265122973Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"0ac2ce2e-de0b-415a-82d4-2fd0354a0d15 is now active with session: 749c37b8-d8e2-44f2-ace9-7d69d329d0b3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265132882Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.26514328Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"session_id":"749c37b8-d8e2-44f2-ace9-7d69d329d0b3"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265152309Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838} {"msg":"Activated sub_volume 0ac2ce2e-de0b-415a-82d4-2fd0354a0d15","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265160428Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265168547Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.265499369Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"task":"crutest"} 01/30 Write block 503 len 16384 data: 14 14 16 15 02/30 Read block 484 len 8192 03/30 Read block 576 len 8192 04/30 Write block 442 len 40960 data: 17 14 16 16 16 14 13 12 13 9 05/30 Read block 123 len 28672 06/30 Read block 26 len 40960 07/30 Read block 296 len 24576 08/30 Read block 110 len 32768 09/30 Read block 122 len 12288 10/30 Read block 86 len 8192 11/30 Read block 232 len 36864 12/30 Write block 381 len 40960 data: 10 13 16 13 12 14 13 10 14 13 13/30 Write block 206 len 28672 data: 12 10 14 17 10 7 7 14/30 Write block 144 len 40960 data: 15 17 15 17 18 16 17 18 17 17 15/30 Read block 397 len 16384 16/30 Read block 116 len 36864 17/30 Write block 171 len 28672 data: 13 13 13 9 9 10 9 18/30 Flush 19/30 Write block 421 len 32768 data: 14 11 10 8 6 11 12 13 20/30 Write block 108 len 16384 data: 12 8 9 10 21/30 Read block 463 len 16384 22/30 Read block 49 len 24576 23/30 Read block 261 len 8192 24/30 Read block 157 len 20480 25/30 Read block 97 len 16384 26/30 Read block 411 len 4096 27/30 Read block 298 len 16384 28/30 Write block 578 len 8192 data: 13 16 29/30 Write block 180 len 4096 data: 15 30/30 Write block 478 len 32768 data: 14 15 11 14 11 11 13 10 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Sent Done Done false 1001 Acked Read 2 Sent Done Done false 1002 Acked Read 2 Sent Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Read 6 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Write 2 Sent Done Sent false 1028 Acked Write 1 Sent Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 3 34 Done 0 29 27 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:27.274275194Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:27.274442535Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1838,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 302 302 302 F F F 1 020-039 172 172 172 301 301 301 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 173 173 173 302 302 302 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 169 169 169 297 297 297 F T T <--- 6 120-139 171 171 171 299 299 299 F F F 7 140-159 171 174 174 299 303 303 F F F <--- 8 160-179 169 174 174 297 303 303 F F F <--- 9 180-199 163 163 163 285 285 285 F T T <--- 10 200-219 168 174 174 296 303 303 F F F <--- 11 220-239 168 168 168 296 296 296 F F F 12 240-259 173 173 173 302 302 302 F F F 13 260-279 172 172 172 300 300 300 F F F 14 280-299 172 172 172 300 300 300 F F F 15 300-319 173 173 173 302 302 302 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 171 171 171 299 299 299 F F F 18 360-379 171 171 171 299 299 299 F F F 19 380-399 171 174 174 299 303 303 F F F <--- 20 400-419 172 172 172 300 300 300 F F F 21 420-439 173 173 173 302 302 302 F T T <--- 22 440-459 173 174 174 302 303 303 F F F <--- 23 460-479 166 166 166 290 290 290 F T T <--- 24 480-499 170 170 170 298 298 298 F T T <--- 25 500-519 162 174 174 284 303 303 F F F <--- 26 520-539 171 171 171 299 299 299 F F F 27 540-559 173 173 173 302 302 302 F F F 28 560-579 170 170 170 298 298 298 F T T <--- 29 580-599 171 171 171 299 299 299 F F F Max gen: 174, Max flush: 303 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 302 302 302 F F F 1 020-039 172 172 172 301 301 301 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 173 173 173 302 302 302 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 175 175 175 304 304 304 F F F 6 120-139 171 171 171 299 299 299 F F F 7 140-159 174 174 174 303 303 303 F F F 8 160-179 174 174 174 303 303 303 F F F 9 180-199 175 175 175 304 304 304 F F F 10 200-219 174 174 174 303 303 303 F F F 11 220-239 168 168 168 296 296 296 F F F 12 240-259 173 173 173 302 302 302 F F F 13 260-279 172 172 172 300 300 300 F F F 14 280-299 172 172 172 300 300 300 F F F 15 300-319 173 173 173 302 302 302 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 171 171 171 299 299 299 F F F 18 360-379 171 171 171 299 299 299 F F F 19 380-399 174 174 174 303 303 303 F F F 20 400-419 172 172 172 300 300 300 F F F 21 420-439 175 175 175 304 304 304 F F F 22 440-459 174 174 174 303 303 303 F F F 23 460-479 175 175 175 304 304 304 F F F 24 480-499 175 175 175 304 304 304 F F F 25 500-519 174 174 174 303 303 303 F F F 26 520-539 171 171 171 299 299 299 F F F 27 540-559 173 173 173 302 302 302 F F F 28 560-579 175 175 175 304 304 304 F F F 29 580-599 171 171 171 299 299 299 F F F Max gen: 175, Max flush: 304 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.528164152Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:31.535317757Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.535372661Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.54223Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.54249354Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.542519927Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.542528476Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.542539834Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846} {"msg":"Crucible 32bdd94e-6888-4b88-a54e-885964be9b8e has session id: db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.542879625Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"Upstairs opts: Upstairs UUID: 32bdd94e-6888-4b88-a54e-885964be9b8e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.542909661Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"Crucible stats registered with UUID: 32bdd94e-6888-4b88-a54e-885964be9b8e","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.54292268Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:31.542932129Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543054824Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543071472Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543081501Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543219235Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543243102Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846} {"msg":"32bdd94e-6888-4b88-a54e-885964be9b8e active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543268779Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543278298Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"client":"0","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543287977Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"client":"1","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543297596Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"client":"2","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543307165Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543317164Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543326922Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:44808)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543452448Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:49170)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543524679Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"ds_connection connected from Ok(127.0.0.1:43185)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.543555876Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544138458Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"client":"1","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544197531Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"client":"0","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544222668Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"client":"2","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544481898Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544515194Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"[0]R flush_numbers[0..12]: [302, 301, 294, 302, 299, 304, 299, 303, 303, 304, 303, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544563958Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"downstairs","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"[0]R generation[0..12]: [173, 172, 168, 173, 171, 175, 171, 174, 174, 175, 174, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544575817Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"downstairs","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544586945Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"downstairs","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"[1]R flush_numbers[0..12]: [302, 301, 294, 302, 299, 304, 299, 303, 303, 304, 303, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544597764Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"downstairs","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"[1]R generation[0..12]: [173, 172, 168, 173, 171, 175, 171, 174, 174, 175, 174, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544608283Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"downstairs","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544619342Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"downstairs","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"[2]R flush_numbers[0..12]: [302, 301, 294, 302, 299, 304, 299, 303, 303, 304, 303, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.54463001Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"downstairs","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"[2]R generation[0..12]: [173, 172, 168, 173, 171, 175, 171, 174, 174, 175, 174, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544640609Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"downstairs","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544653918Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"downstairs","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544663436Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"downstairs","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544673225Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"downstairs","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544682624Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"downstairs","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544692473Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"":"downstairs","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544702432Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544711341Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"client":"0","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.54472118Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"client":"1","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544730869Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"client":"2","":"downstairs","session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544740637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544749776Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"32bdd94e-6888-4b88-a54e-885964be9b8e is now active with session: db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544759755Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544768604Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"session_id":"db5f9ef2-8b79-49a1-8794-e80b1dc7e8a0"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544777923Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846} {"msg":"Activated sub_volume 32bdd94e-6888-4b88-a54e-885964be9b8e","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544785942Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.544793901Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.545105455Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"task":"crutest"} 01/30 Write block 494 len 4096 data: 18 02/30 Write block 466 len 32768 data: 8 8 7 7 4 4 6 9 03/30 Read block 185 len 8192 04/30 Read block 515 len 28672 05/30 Write block 271 len 32768 data: 18 17 18 18 18 17 15 16 06/30 Write block 548 len 20480 data: 22 21 24 23 23 07/30 Read block 376 len 20480 08/30 Read block 128 len 8192 09/30 Write block 152 len 28672 data: 18 18 15 15 15 14 13 10/30 Read block 314 len 20480 11/30 Read block 298 len 40960 12/30 Flush 13/30 Write block 342 len 28672 data: 7 7 7 7 10 10 11 14/30 Write block 157 len 40960 data: 15 14 17 15 11 8 10 11 13 14 15/30 Read block 276 len 24576 16/30 Read block 499 len 28672 17/30 Read block 274 len 32768 18/30 Read block 102 len 32768 19/30 Read block 125 len 32768 20/30 Read block 346 len 16384 21/30 Write block 471 len 8192 data: 5 7 22/30 Read block 540 len 36864 23/30 Write block 111 len 24576 data: 11 9 8 7 9 12 24/30 Read block 7 len 8192 25/30 Read block 309 len 20480 26/30 Read block 35 len 40960 27/30 Read block 269 len 20480 28/30 Read block 364 len 32768 29/30 Read block 588 len 8192 30/30 Read block 216 len 24576 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Sent Done Done false 1001 Acked Write 8 Sent Done Done false 1002 Acked Read 2 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Write 8 Sent Done Done false 1005 Acked Write 5 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1011)) Some(JobId(1011)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:31.554760518Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:31.554940277Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1846,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 302 302 302 F F F 1 020-039 172 172 172 301 301 301 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 173 173 173 302 302 302 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 175 175 175 304 304 304 F T T <--- 6 120-139 171 171 171 299 299 299 F F F 7 140-159 174 176 176 303 305 305 F T T <--- 8 160-179 174 174 174 303 303 303 F T T <--- 9 180-199 175 175 175 304 304 304 F F F 10 200-219 174 174 174 303 303 303 F F F 11 220-239 168 168 168 296 296 296 F F F 12 240-259 173 173 173 302 302 302 F F F 13 260-279 172 176 176 300 305 305 F F F <--- 14 280-299 172 172 172 300 300 300 F F F 15 300-319 173 173 173 302 302 302 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 171 171 171 299 299 299 F T T <--- 18 360-379 171 171 171 299 299 299 F F F 19 380-399 174 174 174 303 303 303 F F F 20 400-419 172 172 172 300 300 300 F F F 21 420-439 175 175 175 304 304 304 F F F 22 440-459 174 174 174 303 303 303 F F F 23 460-479 175 176 176 304 305 305 F T T <--- 24 480-499 175 176 176 304 305 305 F F F <--- 25 500-519 174 174 174 303 303 303 F F F 26 520-539 171 171 171 299 299 299 F F F 27 540-559 173 176 176 302 305 305 F F F <--- 28 560-579 175 175 175 304 304 304 F F F 29 580-599 171 171 171 299 299 299 F F F Max gen: 176, Max flush: 305 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 302 302 302 F F F 1 020-039 172 172 172 301 301 301 F F F 2 040-059 168 168 168 294 294 294 F F F 3 060-079 173 173 173 302 302 302 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 177 177 177 306 306 306 F F F 6 120-139 171 171 171 299 299 299 F F F 7 140-159 177 177 177 306 306 306 F F F 8 160-179 177 177 177 306 306 306 F F F 9 180-199 175 175 175 304 304 304 F F F 10 200-219 174 174 174 303 303 303 F F F 11 220-239 168 168 168 296 296 296 F F F 12 240-259 173 173 173 302 302 302 F F F 13 260-279 176 176 176 305 305 305 F F F 14 280-299 172 172 172 300 300 300 F F F 15 300-319 173 173 173 302 302 302 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 177 177 177 306 306 306 F F F 18 360-379 171 171 171 299 299 299 F F F 19 380-399 174 174 174 303 303 303 F F F 20 400-419 172 172 172 300 300 300 F F F 21 420-439 175 175 175 304 304 304 F F F 22 440-459 174 174 174 303 303 303 F F F 23 460-479 177 177 177 306 306 306 F F F 24 480-499 176 176 176 305 305 305 F F F 25 500-519 174 174 174 303 303 303 F F F 26 520-539 171 171 171 299 299 299 F F F 27 540-559 176 176 176 305 305 305 F F F 28 560-579 175 175 175 304 304 304 F F F 29 580-599 171 171 171 299 299 299 F F F Max gen: 177, Max flush: 306 Begin loop 89 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.802024089Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.810473423Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.810769658Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.810796655Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.810805154Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.810816673Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854} {"msg":"Crucible e53d8e08-0410-4d3e-ae4c-6ef1b69b47cd has session id: a95ddf01-c2aa-466f-a8a3-450f88d325af","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811176731Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"Upstairs opts: Upstairs UUID: e53d8e08-0410-4d3e-ae4c-6ef1b69b47cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811205887Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"Crucible stats registered with UUID: e53d8e08-0410-4d3e-ae4c-6ef1b69b47cd","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811216716Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:35.811225925Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.81135669Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811372008Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811381967Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.81152489Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811540638Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854} {"msg":"e53d8e08-0410-4d3e-ae4c-6ef1b69b47cd active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811564505Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811573894Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"client":"0","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811583483Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"client":"1","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811593142Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"client":"2","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811602731Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.81161276Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811622659Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"ds_connection connected from Ok(127.0.0.1:57845)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811711558Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"ds_connection connected from Ok(127.0.0.1:48487)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.81178547Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"ds_connection connected from Ok(127.0.0.1:54509)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.811807677Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812433204Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"client":"1","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812458631Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"client":"0","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812488688Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"client":"2","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812759156Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812771855Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"[0]R flush_numbers[0..12]: [302, 301, 294, 302, 299, 306, 299, 306, 306, 304, 303, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812787523Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"downstairs","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"[0]R generation[0..12]: [173, 172, 168, 173, 171, 177, 171, 177, 177, 175, 174, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812798351Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"downstairs","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.81280933Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"downstairs","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"[1]R flush_numbers[0..12]: [302, 301, 294, 302, 299, 306, 299, 306, 306, 304, 303, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812820179Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"downstairs","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"[1]R generation[0..12]: [173, 172, 168, 173, 171, 177, 171, 177, 177, 175, 174, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812830868Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"downstairs","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812841686Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"downstairs","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"[2]R flush_numbers[0..12]: [302, 301, 294, 302, 299, 306, 299, 306, 306, 304, 303, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812852335Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"downstairs","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"[2]R generation[0..12]: [173, 172, 168, 173, 171, 177, 171, 177, 177, 175, 174, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812862934Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"downstairs","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812873693Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"downstairs","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812883412Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"downstairs","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.81289319Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"downstairs","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812902649Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"downstairs","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812912128Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"":"downstairs","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812923677Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812932486Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"client":"0","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812942155Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"client":"1","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812951714Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"client":"2","":"downstairs","session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812961622Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812970741Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"e53d8e08-0410-4d3e-ae4c-6ef1b69b47cd is now active with session: a95ddf01-c2aa-466f-a8a3-450f88d325af","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.81298079Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812989619Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"session_id":"a95ddf01-c2aa-466f-a8a3-450f88d325af"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.812998758Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854} {"msg":"Activated sub_volume e53d8e08-0410-4d3e-ae4c-6ef1b69b47cd","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.813006847Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.813014986Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.813313211Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"task":"crutest"} 01/30 Write block 263 len 32768 data: 12 11 9 12 17 19 22 21 02/30 Read block 9 len 36864 03/30 Read block 566 len 4096 04/30 Write block 296 len 12288 data: 17 17 18 05/30 Write block 582 len 40960 data: 14 12 10 12 11 13 11 14 13 10 06/30 Read block 547 len 36864 07/30 Write block 540 len 12288 data: 13 13 14 08/30 Flush 09/30 Write block 75 len 4096 data: 14 10/30 Write block 497 len 32768 data: 12 11 13 14 13 11 15 15 11/30 Write block 136 len 40960 data: 11 10 11 12 13 12 14 15 16 18 12/30 Write block 211 len 32768 data: 8 8 7 6 7 7 8 9 13/30 Read block 80 len 20480 14/30 Flush 15/30 Write block 591 len 36864 data: 11 11 13 10 12 11 11 9 5 16/30 Flush 17/30 Write block 151 len 12288 data: 19 19 19 18/30 Write block 572 len 24576 data: 15 15 16 16 15 16 19/30 Flush 20/30 Flush 21/30 Write block 154 len 28672 data: 16 16 16 16 15 18 16 22/30 Read block 316 len 28672 23/30 Write block 166 len 32768 data: 15 12 13 13 11 14 14 14 24/30 Flush 25/30 Write block 117 len 4096 data: 13 26/30 Write block 125 len 20480 data: 17 17 15 15 17 27/30 Write block 54 len 36864 data: 17 17 21 17 17 15 15 16 15 28/30 Write block 493 len 40960 data: 17 19 17 16 13 12 14 15 14 12 29/30 Read block 14 len 36864 30/30 Read block 125 len 40960 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Write 3 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 27 28 Done 30 29 3 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:35.822378312Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:35.822577099Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1854,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 302 302 302 F F F 1 020-039 172 172 172 301 301 301 F F F 2 040-059 168 168 168 294 294 294 T T F <--- 3 060-079 178 178 173 308 308 302 T T F <--- 4 080-099 171 171 171 299 299 299 F F F 5 100-119 177 177 177 306 306 306 T T F <--- 6 120-139 178 178 171 308 308 299 T T F <--- 7 140-159 178 178 177 312 312 306 F F F <--- 8 160-179 178 178 177 312 312 306 F F F <--- 9 180-199 175 175 175 304 304 304 F F F 10 200-219 178 178 174 308 308 303 F F F <--- 11 220-239 168 168 168 296 296 296 F F F 12 240-259 173 173 173 302 302 302 F F F 13 260-279 178 178 176 307 307 305 F F T <--- 14 280-299 178 178 172 307 307 300 F F F <--- 15 300-319 173 173 173 302 302 302 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 177 177 177 306 306 306 F F F 18 360-379 171 171 171 299 299 299 F F F 19 380-399 174 174 174 303 303 303 F F F 20 400-419 172 172 172 300 300 300 F F F 21 420-439 175 175 175 304 304 304 F F F 22 440-459 174 174 174 303 303 303 F F F 23 460-479 177 177 177 306 306 306 F F F 24 480-499 178 178 176 308 308 305 T T F <--- 25 500-519 178 178 174 308 308 303 T T F <--- 26 520-539 171 171 171 299 299 299 F F F 27 540-559 178 178 176 307 307 305 F F F <--- 28 560-579 178 178 175 310 310 304 F F F <--- 29 580-599 178 178 171 309 309 299 F F F <--- Max gen: 178, Max flush: 312 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 302 302 302 F F F 1 020-039 172 172 172 301 301 301 F F F 2 040-059 179 179 179 313 313 313 F F F 3 060-079 179 179 179 313 313 313 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 179 179 179 313 313 313 F F F 6 120-139 179 179 179 313 313 313 F F F 7 140-159 178 178 178 312 312 312 F F F 8 160-179 178 178 178 312 312 312 F F F 9 180-199 175 175 175 304 304 304 F F F 10 200-219 178 178 178 308 308 308 F F F 11 220-239 168 168 168 296 296 296 F F F 12 240-259 173 173 173 302 302 302 F F F 13 260-279 178 178 178 307 307 307 F F F 14 280-299 178 178 178 307 307 307 F F F 15 300-319 173 173 173 302 302 302 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 177 177 177 306 306 306 F F F 18 360-379 171 171 171 299 299 299 F F F 19 380-399 174 174 174 303 303 303 F F F 20 400-419 172 172 172 300 300 300 F F F 21 420-439 175 175 175 304 304 304 F F F 22 440-459 174 174 174 303 303 303 F F F 23 460-479 177 177 177 306 306 306 F F F 24 480-499 179 179 179 313 313 313 F F F 25 500-519 179 179 179 313 313 313 F F F 26 520-539 171 171 171 299 299 299 F F F 27 540-559 178 178 178 307 307 307 F F F 28 560-579 178 178 178 310 310 310 F F F 29 580-599 178 178 178 309 309 309 F F F Max gen: 179, Max flush: 313 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.075357007Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.083478899Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.083764325Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.083799041Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.08381096Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.083823229Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862} {"msg":"Crucible 4ca4ffdf-6b57-4dcf-95ea-8ecba36e03b8 has session id: 84290044-acd1-4419-9012-7c5f9a4f7068","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084145221Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"Upstairs opts: Upstairs UUID: 4ca4ffdf-6b57-4dcf-95ea-8ecba36e03b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084174138Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"Crucible stats registered with UUID: 4ca4ffdf-6b57-4dcf-95ea-8ecba36e03b8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084185456Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:40.084194745Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.08432414Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084339608Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084349917Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084477762Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.08449348Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862} {"msg":"4ca4ffdf-6b57-4dcf-95ea-8ecba36e03b8 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084527116Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084536675Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"client":"0","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084546284Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"client":"1","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084555953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"client":"2","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084567522Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084577481Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084587329Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"ds_connection connected from Ok(127.0.0.1:60494)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084636534Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"ds_connection connected from Ok(127.0.0.1:50220)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084647712Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"ds_connection connected from Ok(127.0.0.1:33514)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.084766718Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085275619Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"client":"2","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085381577Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"client":"0","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085403894Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"client":"1","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085567735Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085644946Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"[0]R flush_numbers[0..12]: [302, 301, 313, 313, 299, 313, 313, 312, 312, 304, 308, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.08569301Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"downstairs","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"[0]R generation[0..12]: [173, 172, 179, 179, 171, 179, 179, 178, 178, 175, 178, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085704719Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"downstairs","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085715808Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"downstairs","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"[1]R flush_numbers[0..12]: [302, 301, 313, 313, 299, 313, 313, 312, 312, 304, 308, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085729286Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"downstairs","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"[1]R generation[0..12]: [173, 172, 179, 179, 171, 179, 179, 178, 178, 175, 178, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085739915Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"downstairs","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085750844Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"downstairs","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"[2]R flush_numbers[0..12]: [302, 301, 313, 313, 299, 313, 313, 312, 312, 304, 308, 296]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085761352Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"downstairs","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"[2]R generation[0..12]: [173, 172, 179, 179, 171, 179, 179, 178, 178, 175, 178, 168]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085771781Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"downstairs","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.08578266Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"downstairs","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085792169Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"downstairs","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085801998Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"downstairs","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085811387Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"downstairs","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085820885Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"":"downstairs","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085830794Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085839693Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"client":"0","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085849652Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"client":"1","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085859491Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"client":"2","":"downstairs","session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.08586964Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085880138Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"4ca4ffdf-6b57-4dcf-95ea-8ecba36e03b8 is now active with session: 84290044-acd1-4419-9012-7c5f9a4f7068","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085890097Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 30,"time":"2026-09-16T04:45:40.085903596Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"session_id":"84290044-acd1-4419-9012-7c5f9a4f7068"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085912855Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862} {"msg":"Activated sub_volume 4ca4ffdf-6b57-4dcf-95ea-8ecba36e03b8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085920854Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.085928953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.086159506Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"task":"crutest"} 01/30 Write block 175 len 36864 data: 10 11 10 15 15 16 15 13 14 02/30 Read block 340 len 36864 03/30 Read block 328 len 16384 04/30 Write block 460 len 4096 data: 10 05/30 Read block 547 len 12288 06/30 Write block 50 len 32768 data: 11 17 14 15 18 18 22 18 07/30 Read block 468 len 32768 08/30 Read block 409 len 16384 09/30 Read block 491 len 16384 10/30 Read block 264 len 8192 11/30 Write block 277 len 4096 data: 16 12/30 Write block 289 len 28672 data: 14 13 11 9 15 16 16 13/30 Read block 225 len 4096 14/30 Read block 401 len 12288 15/30 Write block 36 len 20480 data: 11 11 13 12 14 16/30 Read block 545 len 20480 17/30 Read block 515 len 40960 18/30 Flush 19/30 Write block 226 len 36864 data: 16 17 14 13 9 11 11 11 11 20/30 Write block 14 len 4096 data: 11 21/30 Read block 323 len 16384 22/30 Write block 504 len 36864 data: 16 17 16 15 10 10 9 10 10 23/30 Read block 587 len 36864 24/30 Write block 258 len 20480 data: 8 8 7 10 11 25/30 Write block 553 len 12288 data: 23 22 24 26/30 Read block 408 len 28672 27/30 Write block 147 len 16384 data: 18 19 17 18 28/30 Read block 300 len 36864 29/30 Read block 57 len 4096 30/30 Read block 196 len 28672 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Sent Done false 1001 Acked Read 9 Done Sent Done false 1002 Acked Read 4 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Read 10 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Read 4 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) None Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:40.095104112Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:40.095285411Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1862,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 173 173 173 302 302 302 T F T <--- 1 020-039 180 172 180 314 301 314 F F F <--- 2 040-059 180 179 180 314 313 314 F F F <--- 3 060-079 179 179 179 313 313 313 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 179 179 179 313 313 313 F F F 6 120-139 179 179 179 313 313 313 F F F 7 140-159 178 178 178 312 312 312 T F T <--- 8 160-179 180 178 180 314 312 314 F F F <--- 9 180-199 180 175 180 314 304 314 F F F <--- 10 200-219 178 178 178 308 308 308 F F F 11 220-239 168 168 168 296 296 296 T F T <--- 12 240-259 173 173 173 302 302 302 T F T <--- 13 260-279 180 178 180 314 307 314 T F T <--- 14 280-299 180 178 180 314 307 314 F F F <--- 15 300-319 173 173 173 302 302 302 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 177 177 177 306 306 306 F F F 18 360-379 171 171 171 299 299 299 F F F 19 380-399 174 174 174 303 303 303 F F F 20 400-419 172 172 172 300 300 300 F F F 21 420-439 175 175 175 304 304 304 F F F 22 440-459 174 174 174 303 303 303 F F F 23 460-479 180 177 180 314 306 314 F F F <--- 24 480-499 179 179 179 313 313 313 F F F 25 500-519 179 179 179 313 313 313 T F T <--- 26 520-539 171 171 171 299 299 299 F F F 27 540-559 178 178 178 307 307 307 T F T <--- 28 560-579 178 178 178 310 310 310 F F F 29 580-599 178 178 178 309 309 309 F F F Max gen: 180, Max flush: 314 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 315 315 315 F F F 1 020-039 180 180 180 314 314 314 F F F 2 040-059 180 180 180 314 314 314 F F F 3 060-079 179 179 179 313 313 313 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 179 179 179 313 313 313 F F F 6 120-139 179 179 179 313 313 313 F F F 7 140-159 181 181 181 315 315 315 F F F 8 160-179 180 180 180 314 314 314 F F F 9 180-199 180 180 180 314 314 314 F F F 10 200-219 178 178 178 308 308 308 F F F 11 220-239 181 181 181 315 315 315 F F F 12 240-259 181 181 181 315 315 315 F F F 13 260-279 181 181 181 315 315 315 F F F 14 280-299 180 180 180 314 314 314 F F F 15 300-319 173 173 173 302 302 302 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 177 177 177 306 306 306 F F F 18 360-379 171 171 171 299 299 299 F F F 19 380-399 174 174 174 303 303 303 F F F 20 400-419 172 172 172 300 300 300 F F F 21 420-439 175 175 175 304 304 304 F F F 22 440-459 174 174 174 303 303 303 F F F 23 460-479 180 180 180 314 314 314 F F F 24 480-499 179 179 179 313 313 313 F F F 25 500-519 181 181 181 315 315 315 F F F 26 520-539 171 171 171 299 299 299 F F F 27 540-559 181 181 181 315 315 315 F F F 28 560-579 178 178 178 310 310 310 F F F 29 580-599 178 178 178 309 309 309 F F F Max gen: 181, Max flush: 315 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.346193427Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.354328867Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.354604665Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.354630022Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.354638451Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.354649959Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870} {"msg":"Crucible 5fcc0ce1-ed33-4c29-b783-f7b312780184 has session id: b7295fb0-2b47-4af0-9d61-542bbf9e3bdd","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.35498698Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"Upstairs opts: Upstairs UUID: 5fcc0ce1-ed33-4c29-b783-f7b312780184, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.35507455Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"Crucible stats registered with UUID: 5fcc0ce1-ed33-4c29-b783-f7b312780184","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355102777Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:44.355113335Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355235731Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355265528Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355276346Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355406671Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355422569Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870} {"msg":"5fcc0ce1-ed33-4c29-b783-f7b312780184 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355440247Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355449116Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"client":"0","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355458995Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"client":"1","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355468824Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"client":"2","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355478383Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355488452Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.35549825Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:40907)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355563513Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:43358)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.35558943Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"ds_connection connected from Ok(127.0.0.1:41847)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.355678039Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356224416Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"client":"2","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356278829Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"client":"1","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356290478Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"client":"0","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356562786Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356600292Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"[0]R flush_numbers[0..12]: [315, 314, 314, 313, 299, 313, 313, 315, 314, 314, 308, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356639617Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"downstairs","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"[0]R generation[0..12]: [181, 180, 180, 179, 171, 179, 179, 181, 180, 180, 178, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356651716Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"downstairs","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356662824Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"downstairs","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"[1]R flush_numbers[0..12]: [315, 314, 314, 313, 299, 313, 313, 315, 314, 314, 308, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356673653Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"downstairs","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"[1]R generation[0..12]: [181, 180, 180, 179, 171, 179, 179, 181, 180, 180, 178, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356684292Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"downstairs","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356695391Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"downstairs","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"[2]R flush_numbers[0..12]: [315, 314, 314, 313, 299, 313, 313, 315, 314, 314, 308, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356706049Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"downstairs","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"[2]R generation[0..12]: [181, 180, 180, 179, 171, 179, 179, 181, 180, 180, 178, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356716748Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"downstairs","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356735666Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"downstairs","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356745105Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"downstairs","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356754954Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"downstairs","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356764363Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"downstairs","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356773742Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"":"downstairs","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.35678345Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356792389Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"client":"0","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356802048Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"client":"1","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356811807Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"client":"2","":"downstairs","session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356821646Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356830645Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"5fcc0ce1-ed33-4c29-b783-f7b312780184 is now active with session: b7295fb0-2b47-4af0-9d61-542bbf9e3bdd","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356840704Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356849613Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"session_id":"b7295fb0-2b47-4af0-9d61-542bbf9e3bdd"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356858682Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870} {"msg":"Activated sub_volume 5fcc0ce1-ed33-4c29-b783-f7b312780184","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.356866741Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.35687484Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.357188073Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"task":"crutest"} 01/30 Write block 43 len 16384 data: 13 14 15 16 02/30 Read block 246 len 32768 03/30 Flush 04/30 Write block 482 len 24576 data: 12 12 14 11 11 11 05/30 Write block 509 len 12288 data: 11 10 11 06/30 Read block 387 len 28672 07/30 Write block 282 len 36864 data: 12 9 11 11 10 13 12 15 14 08/30 Write block 100 len 36864 data: 9 8 8 8 8 9 14 12 13 09/30 Write block 101 len 24576 data: 9 9 9 9 10 15 10/30 Write block 532 len 16384 data: 15 17 16 17 11/30 Read block 521 len 24576 12/30 Read block 465 len 4096 13/30 Write block 18 len 36864 data: 11 13 12 12 13 13 16 15 20 14/30 Read block 219 len 40960 15/30 Flush 16/30 Write block 284 len 4096 data: 12 17/30 Read block 20 len 36864 18/30 Read block 555 len 28672 19/30 Flush 20/30 Write block 511 len 16384 data: 12 11 10 10 21/30 Write block 524 len 12288 data: 13 11 6 22/30 Write block 470 len 8192 data: 5 6 23/30 Read block 465 len 8192 24/30 Write block 447 len 36864 data: 15 14 13 14 10 8 7 8 7 25/30 Read block 177 len 40960 26/30 Read block 90 len 40960 27/30 Read block 579 len 12288 28/30 Write block 526 len 12288 data: 7 7 8 29/30 Read block 274 len 8192 30/30 Read block 338 len 40960 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 6 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Write 9 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Read 1 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Read 7 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Read 10 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 30 2 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:44.366172094Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:44.366359423Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1870,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 181 182 317 315 317 F F F <--- 1 020-039 182 180 182 317 314 317 F F F <--- 2 040-059 182 180 182 316 314 316 F T F <--- 3 060-079 179 179 179 313 313 313 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 182 179 182 317 313 317 F F F <--- 6 120-139 179 179 179 313 313 313 F F F 7 140-159 181 181 181 315 315 315 F F F 8 160-179 180 180 180 314 314 314 F F F 9 180-199 180 180 180 314 314 314 F F F 10 200-219 178 178 178 308 308 308 F F F 11 220-239 181 181 181 315 315 315 F F F 12 240-259 181 181 181 315 315 315 F F F 13 260-279 181 181 181 315 315 315 F F F 14 280-299 182 180 182 318 314 318 F F F <--- 15 300-319 173 173 173 302 302 302 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 177 177 177 306 306 306 F F F 18 360-379 171 171 171 299 299 299 F F F 19 380-399 174 174 174 303 303 303 F F F 20 400-419 172 172 172 300 300 300 F F F 21 420-439 175 175 175 304 304 304 F F F 22 440-459 174 174 174 303 303 303 T F T <--- 23 460-479 180 180 180 314 314 314 T F T <--- 24 480-499 182 179 182 317 313 317 F F F <--- 25 500-519 182 181 182 317 315 317 T F T <--- 26 520-539 182 171 182 317 299 317 T F T <--- 27 540-559 181 181 181 315 315 315 F F F 28 560-579 178 178 178 310 310 310 F F F 29 580-599 178 178 178 309 309 309 F F F Max gen: 182, Max flush: 318 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 317 317 317 F F F 1 020-039 182 182 182 317 317 317 F F F 2 040-059 182 182 182 316 316 316 F F F 3 060-079 179 179 179 313 313 313 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 182 182 182 317 317 317 F F F 6 120-139 179 179 179 313 313 313 F F F 7 140-159 181 181 181 315 315 315 F F F 8 160-179 180 180 180 314 314 314 F F F 9 180-199 180 180 180 314 314 314 F F F 10 200-219 178 178 178 308 308 308 F F F 11 220-239 181 181 181 315 315 315 F F F 12 240-259 181 181 181 315 315 315 F F F 13 260-279 181 181 181 315 315 315 F F F 14 280-299 182 182 182 318 318 318 F F F 15 300-319 173 173 173 302 302 302 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 177 177 177 306 306 306 F F F 18 360-379 171 171 171 299 299 299 F F F 19 380-399 174 174 174 303 303 303 F F F 20 400-419 172 172 172 300 300 300 F F F 21 420-439 175 175 175 304 304 304 F F F 22 440-459 183 183 183 319 319 319 F F F 23 460-479 183 183 183 319 319 319 F F F 24 480-499 182 182 182 317 317 317 F F F 25 500-519 183 183 183 319 319 319 F F F 26 520-539 183 183 183 319 319 319 F F F 27 540-559 181 181 181 315 315 315 F F F 28 560-579 178 178 178 310 310 310 F F F 29 580-599 178 178 178 309 309 309 F F F Max gen: 183, Max flush: 319 Begin loop 92 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.610614285Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.618771853Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619062969Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619088946Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619097795Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619109233Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878} {"msg":"Crucible e3722651-87a1-4be1-a0ea-0e4eda44c908 has session id: 38dc8d79-615f-411b-af5f-f5cadbffdba5","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619466502Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"Upstairs opts: Upstairs UUID: e3722651-87a1-4be1-a0ea-0e4eda44c908, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619495458Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"Crucible stats registered with UUID: e3722651-87a1-4be1-a0ea-0e4eda44c908","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619506357Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:48.619515706Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.61965363Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619682667Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619693985Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619793384Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619807102Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878} {"msg":"e3722651-87a1-4be1-a0ea-0e4eda44c908 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.61982411Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619833039Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"client":"0","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619842788Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"client":"1","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619852357Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"client":"2","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619861876Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619871854Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619881623Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"ds_connection connected from Ok(127.0.0.1:36080)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.619977672Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"ds_connection connected from Ok(127.0.0.1:62509)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.6199971Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"ds_connection connected from Ok(127.0.0.1:51549)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.620008099Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.620604919Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"client":"0","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.620643864Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"client":"2","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.620761361Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"client":"1","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.620927681Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.620948329Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"[0]R flush_numbers[0..12]: [317, 317, 316, 313, 299, 317, 313, 315, 314, 314, 308, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621030559Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"downstairs","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"[0]R generation[0..12]: [182, 182, 182, 179, 171, 182, 179, 181, 180, 180, 178, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621041978Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"downstairs","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621052977Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"downstairs","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"[1]R flush_numbers[0..12]: [317, 317, 316, 313, 299, 317, 313, 315, 314, 314, 308, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621063645Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"downstairs","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"[1]R generation[0..12]: [182, 182, 182, 179, 171, 182, 179, 181, 180, 180, 178, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621074104Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"downstairs","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621084943Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"downstairs","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"[2]R flush_numbers[0..12]: [317, 317, 316, 313, 299, 317, 313, 315, 314, 314, 308, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621095292Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"downstairs","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"[2]R generation[0..12]: [182, 182, 182, 179, 171, 182, 179, 181, 180, 180, 178, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.62110579Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"downstairs","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621116919Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"downstairs","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621126448Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"downstairs","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621136147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"downstairs","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621145456Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"downstairs","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621155035Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"":"downstairs","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621166433Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621175222Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"client":"0","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621184991Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"client":"1","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.62119478Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"client":"2","":"downstairs","session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621204549Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621213618Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"e3722651-87a1-4be1-a0ea-0e4eda44c908 is now active with session: 38dc8d79-615f-411b-af5f-f5cadbffdba5","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621223517Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621232176Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"session_id":"38dc8d79-615f-411b-af5f-f5cadbffdba5"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621241345Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878} {"msg":"Activated sub_volume e3722651-87a1-4be1-a0ea-0e4eda44c908","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621249514Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621261382Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.621513793Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"task":"crutest"} 01/30 Write block 288 len 20480 data: 13 16 15 12 10 02/30 Read block 131 len 4096 03/30 Read block 423 len 40960 04/30 Flush 05/30 Write block 462 len 36864 data: 8 8 9 8 9 9 8 8 6 06/30 Write block 472 len 28672 data: 8 10 11 11 14 14 15 07/30 Read block 474 len 8192 08/30 Write block 570 len 12288 data: 13 15 16 09/30 Flush 10/30 Flush 11/30 Flush 12/30 Write block 284 len 4096 data: 13 13/30 Read block 563 len 8192 14/30 Read block 37 len 12288 15/30 Write block 350 len 28672 data: 11 16 17 19 20 16 13 16/30 Read block 206 len 12288 17/30 Write block 456 len 8192 data: 9 11 18/30 Write block 116 len 32768 data: 13 14 16 16 16 16 16 16 19/30 Read block 471 len 20480 20/30 Flush 21/30 Flush 22/30 Write block 148 len 8192 data: 20 18 23/30 Write block 112 len 24576 data: 10 9 8 10 14 15 24/30 Write block 3 len 4096 data: 12 25/30 Write block 143 len 20480 data: 16 17 19 16 19 26/30 Flush 27/30 Write block 33 len 4096 data: 18 28/30 Read block 36 len 4096 29/30 Write block 396 len 24576 data: 15 15 17 14 15 12 30/30 Write block 446 len 20480 data: 17 16 15 14 15 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 10 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Write 7 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 1 Done Sent Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 28 2 32 Done 28 2 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:48.629238381Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:48.629429749Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1878,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 182 184 326 317 326 F F F <--- 1 020-039 182 182 182 317 317 317 T F T <--- 2 040-059 182 182 182 316 316 316 F F F 3 060-079 179 179 179 313 313 313 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 184 182 184 326 317 326 F F F <--- 6 120-139 184 179 184 324 313 324 F F F <--- 7 140-159 184 181 184 326 315 326 F F F <--- 8 160-179 180 180 180 314 314 314 F F F 9 180-199 180 180 180 314 314 314 F F F 10 200-219 178 178 178 308 308 308 F F F 11 220-239 181 181 181 315 315 315 F F F 12 240-259 181 181 181 315 315 315 F F F 13 260-279 181 181 181 315 315 315 F F F 14 280-299 184 182 184 324 318 324 F T F <--- 15 300-319 173 173 173 302 302 302 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 184 177 184 324 306 324 F F F <--- 18 360-379 171 171 171 299 299 299 F F F 19 380-399 174 174 174 303 303 303 T F T <--- 20 400-419 172 172 172 300 300 300 T F T <--- 21 420-439 175 175 175 304 304 304 F F F 22 440-459 184 183 184 324 319 324 T F T <--- 23 460-479 184 183 184 321 319 321 F F F <--- 24 480-499 182 182 182 317 317 317 F F F 25 500-519 183 183 183 319 319 319 F F F 26 520-539 183 183 183 319 319 319 F F F 27 540-559 181 181 181 315 315 315 F F F 28 560-579 184 178 184 321 310 321 F F F <--- 29 580-599 178 178 178 309 309 309 F F F Max gen: 184, Max flush: 326 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 326 326 326 F F F 1 020-039 185 185 185 327 327 327 F F F 2 040-059 182 182 182 316 316 316 F F F 3 060-079 179 179 179 313 313 313 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 184 184 184 326 326 326 F F F 6 120-139 184 184 184 324 324 324 F F F 7 140-159 184 184 184 326 326 326 F F F 8 160-179 180 180 180 314 314 314 F F F 9 180-199 180 180 180 314 314 314 F F F 10 200-219 178 178 178 308 308 308 F F F 11 220-239 181 181 181 315 315 315 F F F 12 240-259 181 181 181 315 315 315 F F F 13 260-279 181 181 181 315 315 315 F F F 14 280-299 184 184 184 324 324 324 F F F 15 300-319 173 173 173 302 302 302 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 184 184 184 324 324 324 F F F 18 360-379 171 171 171 299 299 299 F F F 19 380-399 185 185 185 327 327 327 F F F 20 400-419 185 185 185 327 327 327 F F F 21 420-439 175 175 175 304 304 304 F F F 22 440-459 185 185 185 327 327 327 F F F 23 460-479 184 184 184 321 321 321 F F F 24 480-499 182 182 182 317 317 317 F F F 25 500-519 183 183 183 319 319 319 F F F 26 520-539 183 183 183 319 319 319 F F F 27 540-559 181 181 181 315 315 315 F F F 28 560-579 184 184 184 321 321 321 F F F 29 580-599 178 178 178 309 309 309 F F F Max gen: 185, Max flush: 327 Begin loop 93 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.879951779Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.888137204Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.88842591Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.888455547Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.888466675Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.888478404Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886} {"msg":"Crucible 3461dc77-6834-4e47-8471-386d16c5fc09 has session id: a7b2ec2f-d261-4a82-80ec-a5d6c9c36683","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.88876927Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"Upstairs opts: Upstairs UUID: 3461dc77-6834-4e47-8471-386d16c5fc09, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.888785438Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"Crucible stats registered with UUID: 3461dc77-6834-4e47-8471-386d16c5fc09","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.888795697Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:52.888805006Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.888950879Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.888965677Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.888975746Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.889117479Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.889135377Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886} {"msg":"3461dc77-6834-4e47-8471-386d16c5fc09 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.889169753Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.889179272Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"client":"0","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.889188991Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"client":"1","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.88919856Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"client":"2","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.889208109Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.889246294Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.889268892Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"ds_connection connected from Ok(127.0.0.1:65408)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.889361781Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"ds_connection connected from Ok(127.0.0.1:35549)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.889490936Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"ds_connection connected from Ok(127.0.0.1:34018)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.889503814Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.889959141Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"client":"1","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890005206Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"client":"0","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890175496Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"client":"2","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890294322Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.89031399Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"[0]R flush_numbers[0..12]: [326, 327, 316, 313, 299, 326, 324, 326, 314, 314, 308, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890442115Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"downstairs","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"[0]R generation[0..12]: [184, 185, 182, 179, 171, 184, 184, 184, 180, 180, 178, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890453723Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"downstairs","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890464712Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"downstairs","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"[1]R flush_numbers[0..12]: [326, 327, 316, 313, 299, 326, 324, 326, 314, 314, 308, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890475501Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"downstairs","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"[1]R generation[0..12]: [184, 185, 182, 179, 171, 184, 184, 184, 180, 180, 178, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890488329Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"downstairs","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890499398Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"downstairs","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"[2]R flush_numbers[0..12]: [326, 327, 316, 313, 299, 326, 324, 326, 314, 314, 308, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890509927Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"downstairs","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"[2]R generation[0..12]: [184, 185, 182, 179, 171, 184, 184, 184, 180, 180, 178, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890520396Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"downstairs","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890531264Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"downstairs","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890540673Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"downstairs","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890550402Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"downstairs","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890559751Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"downstairs","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.89056908Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"":"downstairs","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890578849Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890587538Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"client":"0","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890597147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"client":"1","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890606825Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"client":"2","":"downstairs","session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890616584Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890625593Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"3461dc77-6834-4e47-8471-386d16c5fc09 is now active with session: a7b2ec2f-d261-4a82-80ec-a5d6c9c36683","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890642511Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.89065123Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"session_id":"a7b2ec2f-d261-4a82-80ec-a5d6c9c36683"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890660249Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886} {"msg":"Activated sub_volume 3461dc77-6834-4e47-8471-386d16c5fc09","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890668288Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890676507Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.890889382Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"task":"crutest"} 01/30 Write block 310 len 16384 data: 16 10 10 10 02/30 Write block 215 len 4096 data: 8 03/30 Read block 485 len 36864 04/30 Read block 578 len 24576 05/30 Write block 16 len 12288 data: 11 12 12 06/30 Write block 277 len 4096 data: 17 07/30 Read block 201 len 16384 08/30 Read block 405 len 24576 09/30 Flush 10/30 Write block 161 len 4096 data: 12 11/30 Flush 12/30 Write block 574 len 8192 data: 17 17 13/30 Read block 390 len 32768 14/30 Read block 532 len 40960 15/30 Write block 30 len 36864 data: 22 24 22 19 14 12 12 12 14 16/30 Read block 476 len 36864 17/30 Read block 115 len 24576 18/30 Read block 568 len 36864 19/30 Read block 461 len 28672 20/30 Write block 398 len 24576 data: 18 15 16 13 9 10 21/30 Flush 22/30 Write block 478 len 40960 data: 16 16 12 15 13 13 15 12 12 12 23/30 Read block 591 len 24576 24/30 Write block 189 len 40960 data: 15 17 17 14 13 13 12 11 11 13 25/30 Read block 582 len 8192 26/30 Write block 371 len 40960 data: 8 9 8 8 10 13 12 10 10 11 27/30 Write block 520 len 36864 data: 13 14 13 14 14 12 8 8 9 28/30 Write block 283 len 36864 data: 10 14 12 11 14 14 17 16 13 29/30 Write block 457 len 20480 data: 12 12 11 11 8 30/30 Read block 388 len 40960 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Read 4 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Write 10 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 24 25 Done 29 30 6 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:52.900453066Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:52.900672101Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1886,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 184 328 328 326 F F T <--- 1 020-039 186 186 185 330 330 327 F F F <--- 2 040-059 182 182 182 316 316 316 F F F 3 060-079 179 179 179 313 313 313 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 184 184 184 326 326 326 F F F 6 120-139 184 184 184 324 324 324 F F F 7 140-159 184 184 184 326 326 326 F F F 8 160-179 186 186 180 329 329 314 F F F <--- 9 180-199 180 180 180 314 314 314 T T F <--- 10 200-219 186 186 178 328 328 308 F F T <--- 11 220-239 181 181 181 315 315 315 F F F 12 240-259 181 181 181 315 315 315 F F F 13 260-279 186 186 181 328 328 315 F F T <--- 14 280-299 184 184 184 324 324 324 T T F <--- 15 300-319 186 186 173 328 328 302 F F T <--- 16 320-339 173 173 173 302 302 302 F F F 17 340-359 184 184 184 324 324 324 F F F 18 360-379 171 171 171 299 299 299 T T F <--- 19 380-399 186 186 185 330 330 327 T T F <--- 20 400-419 186 186 185 330 330 327 F F F <--- 21 420-439 175 175 175 304 304 304 F F F 22 440-459 185 185 185 327 327 327 T T F <--- 23 460-479 184 184 184 321 321 321 T T F <--- 24 480-499 182 182 182 317 317 317 T T F <--- 25 500-519 183 183 183 319 319 319 F F F 26 520-539 183 183 183 319 319 319 T T F <--- 27 540-559 181 181 181 315 315 315 F F F 28 560-579 186 186 184 330 330 321 F F F <--- 29 580-599 178 178 178 309 309 309 F F F Max gen: 186, Max flush: 330 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 328 328 328 F F F 1 020-039 186 186 186 330 330 330 F F F 2 040-059 182 182 182 316 316 316 F F F 3 060-079 179 179 179 313 313 313 F F F 4 080-099 171 171 171 299 299 299 F F F 5 100-119 184 184 184 326 326 326 F F F 6 120-139 184 184 184 324 324 324 F F F 7 140-159 184 184 184 326 326 326 F F F 8 160-179 186 186 186 329 329 329 F F F 9 180-199 187 187 187 331 331 331 F F F 10 200-219 186 186 186 328 328 328 F F F 11 220-239 181 181 181 315 315 315 F F F 12 240-259 181 181 181 315 315 315 F F F 13 260-279 186 186 186 328 328 328 F F F 14 280-299 187 187 187 331 331 331 F F F 15 300-319 186 186 186 328 328 328 F F F 16 320-339 173 173 173 302 302 302 F F F 17 340-359 184 184 184 324 324 324 F F F 18 360-379 187 187 187 331 331 331 F F F 19 380-399 187 187 187 331 331 331 F F F 20 400-419 186 186 186 330 330 330 F F F 21 420-439 175 175 175 304 304 304 F F F 22 440-459 187 187 187 331 331 331 F F F 23 460-479 187 187 187 331 331 331 F F F 24 480-499 187 187 187 331 331 331 F F F 25 500-519 183 183 183 319 319 319 F F F 26 520-539 187 187 187 331 331 331 F F F 27 540-559 181 181 181 315 315 315 F F F 28 560-579 186 186 186 330 330 330 F F F 29 580-599 178 178 178 309 309 309 F F F Max gen: 187, Max flush: 331 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.158195224Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.166488826Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.166771663Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.16679587Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.166804869Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.166816358Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894} {"msg":"Crucible db5ec58b-9922-4f5d-9f80-802709a970f4 has session id: fdcf1962-712d-46f4-98fb-3ffa6ce10aa9","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167156058Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"Upstairs opts: Upstairs UUID: db5ec58b-9922-4f5d-9f80-802709a970f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167183825Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"Crucible stats registered with UUID: db5ec58b-9922-4f5d-9f80-802709a970f4","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167196404Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:57.167205552Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167364614Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167392931Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167403709Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167549732Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.16756897Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894} {"msg":"db5ec58b-9922-4f5d-9f80-802709a970f4 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167592597Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167602046Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"client":"0","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167611695Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"client":"1","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167621294Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"client":"2","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167630813Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167640752Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167650621Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:45595)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167691806Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:59591)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167702974Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33886)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.167713473Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168346639Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"client":"1","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168372336Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"client":"0","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168384035Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"client":"2","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168718116Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168727895Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"[0]R flush_numbers[0..12]: [328, 330, 316, 313, 299, 326, 324, 326, 329, 331, 328, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168763311Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"downstairs","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"[0]R generation[0..12]: [186, 186, 182, 179, 171, 184, 184, 184, 186, 187, 186, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168774659Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"downstairs","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168785578Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"downstairs","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"[1]R flush_numbers[0..12]: [328, 330, 316, 313, 299, 326, 324, 326, 329, 331, 328, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168796237Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"downstairs","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"[1]R generation[0..12]: [186, 186, 182, 179, 171, 184, 184, 184, 186, 187, 186, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168806816Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"downstairs","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168817554Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"downstairs","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"[2]R flush_numbers[0..12]: [328, 330, 316, 313, 299, 326, 324, 326, 329, 331, 328, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168828133Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"downstairs","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"[2]R generation[0..12]: [186, 186, 182, 179, 171, 184, 184, 184, 186, 187, 186, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168838522Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"downstairs","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.16885116Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"downstairs","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168860779Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"downstairs","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168870568Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"downstairs","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168879897Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"downstairs","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168889276Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"":"downstairs","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168899155Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168908024Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"client":"0","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168917843Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"client":"1","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168927631Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"client":"2","":"downstairs","session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.16893759Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168946919Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"db5ec58b-9922-4f5d-9f80-802709a970f4 is now active with session: fdcf1962-712d-46f4-98fb-3ffa6ce10aa9","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168956928Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168965637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"session_id":"fdcf1962-712d-46f4-98fb-3ffa6ce10aa9"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168974686Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894} {"msg":"Activated sub_volume db5ec58b-9922-4f5d-9f80-802709a970f4","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-09-16T04:45:57.168982785Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.168994944Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.169243115Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"task":"crutest"} 01/30 Write block 430 len 16384 data: 13 14 15 16 02/30 Flush 03/30 Write block 263 len 12288 data: 13 12 10 04/30 Flush 05/30 Write block 337 len 32768 data: 16 12 13 12 9 8 8 8 06/30 Write block 345 len 36864 data: 8 11 11 12 12 12 17 18 20 07/30 Write block 327 len 8192 data: 11 12 08/30 Write block 241 len 40960 data: 19 20 17 19 16 16 16 17 14 17 09/30 Write block 392 len 36864 data: 14 14 15 14 16 16 19 16 17 10/30 Write block 507 len 24576 data: 16 11 12 11 13 12 11/30 Write block 354 len 16384 data: 21 17 14 11 12/30 Write block 266 len 36864 data: 13 18 20 23 22 19 18 19 19 13/30 Write block 561 len 16384 data: 12 13 14 16 14/30 Read block 265 len 8192 15/30 Read block 420 len 8192 16/30 Write block 406 len 28672 data: 9 11 11 12 13 15 15 17/30 Read block 254 len 40960 18/30 Read block 98 len 12288 19/30 Write block 81 len 36864 data: 10 13 16 15 15 17 15 16 20 20/30 Read block 414 len 24576 21/30 Read block 282 len 40960 22/30 Write block 261 len 24576 data: 11 12 14 13 11 14 23/30 Write block 351 len 40960 data: 18 19 21 22 18 15 12 13 13 14 24/30 Read block 545 len 4096 25/30 Read block 35 len 32768 26/30 Write block 115 len 20480 data: 11 15 16 17 17 27/30 Read block 254 len 40960 28/30 Write block 126 len 28672 data: 18 16 16 18 17 15 14 29/30 Write block 52 len 40960 data: 15 16 19 19 23 19 18 16 16 17 30/30 Write block 243 len 28672 data: 18 20 17 17 17 18 15 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 3 Done Done Done false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 8 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Write 9 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Write 4 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Read 8 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 27 31 Done 26 26 1 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:45:57.177197686Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:45:57.177374955Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1894,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 328 328 328 F F F 1 020-039 186 186 186 330 330 330 F F F 2 040-059 182 182 182 316 316 316 T T F <--- 3 060-079 179 179 179 313 313 313 T T F <--- 4 080-099 171 171 171 299 299 299 T T F <--- 5 100-119 184 184 184 326 326 326 T T F <--- 6 120-139 184 184 184 324 324 324 T T F <--- 7 140-159 184 184 184 326 326 326 F F F 8 160-179 186 186 186 329 329 329 F F F 9 180-199 187 187 187 331 331 331 F F F 10 200-219 186 186 186 328 328 328 F F F 11 220-239 181 181 181 315 315 315 F F F 12 240-259 181 181 181 315 315 315 T T F <--- 13 260-279 188 188 186 333 333 328 T T T <--- 14 280-299 187 187 187 331 331 331 F F F 15 300-319 186 186 186 328 328 328 F F F 16 320-339 173 173 173 302 302 302 T T F <--- 17 340-359 184 184 184 324 324 324 T T F <--- 18 360-379 187 187 187 331 331 331 T T F <--- 19 380-399 187 187 187 331 331 331 T T F <--- 20 400-419 186 186 186 330 330 330 T T F <--- 21 420-439 188 188 188 332 332 332 F F F 22 440-459 187 187 187 331 331 331 F F F 23 460-479 187 187 187 331 331 331 F F F 24 480-499 187 187 187 331 331 331 F F F 25 500-519 183 183 183 319 319 319 T T F <--- 26 520-539 187 187 187 331 331 331 F F F 27 540-559 181 181 181 315 315 315 F F F 28 560-579 186 186 186 330 330 330 T T F <--- 29 580-599 178 178 178 309 309 309 F F F Max gen: 188, Max flush: 333 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 328 328 328 F F F 1 020-039 186 186 186 330 330 330 F F F 2 040-059 189 189 189 334 334 334 F F F 3 060-079 189 189 189 334 334 334 F F F 4 080-099 189 189 189 334 334 334 F F F 5 100-119 189 189 189 334 334 334 F F F 6 120-139 189 189 189 334 334 334 F F F 7 140-159 184 184 184 326 326 326 F F F 8 160-179 186 186 186 329 329 329 F F F 9 180-199 187 187 187 331 331 331 F F F 10 200-219 186 186 186 328 328 328 F F F 11 220-239 181 181 181 315 315 315 F F F 12 240-259 189 189 189 334 334 334 F F F 13 260-279 189 189 189 334 334 334 F F F 14 280-299 187 187 187 331 331 331 F F F 15 300-319 186 186 186 328 328 328 F F F 16 320-339 189 189 189 334 334 334 F F F 17 340-359 189 189 189 334 334 334 F F F 18 360-379 189 189 189 334 334 334 F F F 19 380-399 189 189 189 334 334 334 F F F 20 400-419 189 189 189 334 334 334 F F F 21 420-439 188 188 188 332 332 332 F F F 22 440-459 187 187 187 331 331 331 F F F 23 460-479 187 187 187 331 331 331 F F F 24 480-499 187 187 187 331 331 331 F F F 25 500-519 189 189 189 334 334 334 F F F 26 520-539 187 187 187 331 331 331 F F F 27 540-559 181 181 181 315 315 315 F F F 28 560-579 189 189 189 334 334 334 F F F 29 580-599 178 178 178 309 309 309 F F F Max gen: 189, Max flush: 334 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.432385003Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.440588735Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.440895999Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.440955002Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.44097063Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.440982879Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902} {"msg":"Crucible 39531026-a26d-49db-8499-cd766714f019 has session id: cc32d12c-7460-4617-a30b-7a8b5817b7e8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441277744Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"Upstairs opts: Upstairs UUID: 39531026-a26d-49db-8499-cd766714f019, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441302822Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"Crucible stats registered with UUID: 39531026-a26d-49db-8499-cd766714f019","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.4413136Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:46:01.441323009Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441463563Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441492959Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441504008Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441647801Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.44166218Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902} {"msg":"39531026-a26d-49db-8499-cd766714f019 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441697405Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441707164Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"client":"0","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441716863Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"client":"1","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441726562Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"client":"2","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441736291Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.44174614Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441756149Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:41384)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441799874Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:37207)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441810952Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:56015)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.441821301Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.442554685Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"client":"0","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.442576553Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"client":"1","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.442588132Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"client":"2","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.442887207Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.442897115Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"[0]R flush_numbers[0..12]: [328, 330, 334, 334, 334, 334, 334, 326, 329, 331, 328, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.442932451Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"downstairs","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"[0]R generation[0..12]: [186, 186, 189, 189, 189, 189, 189, 184, 186, 187, 186, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.44294392Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"downstairs","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.442955179Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"downstairs","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"[1]R flush_numbers[0..12]: [328, 330, 334, 334, 334, 334, 334, 326, 329, 331, 328, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.442966067Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"downstairs","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"[1]R generation[0..12]: [186, 186, 189, 189, 189, 189, 189, 184, 186, 187, 186, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.442976596Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"downstairs","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.442987575Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"downstairs","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"[2]R flush_numbers[0..12]: [328, 330, 334, 334, 334, 334, 334, 326, 329, 331, 328, 315]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.442998244Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"downstairs","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"[2]R generation[0..12]: [186, 186, 189, 189, 189, 189, 189, 184, 186, 187, 186, 181]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443008842Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"downstairs","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443019861Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"downstairs","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.44302952Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"downstairs","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443039269Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"downstairs","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443048528Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"downstairs","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443058027Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"":"downstairs","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443069315Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443078084Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"client":"0","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443087963Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"client":"1","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443097832Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"client":"2","":"downstairs","session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443108031Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.44311736Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"39531026-a26d-49db-8499-cd766714f019 is now active with session: cc32d12c-7460-4617-a30b-7a8b5817b7e8","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443127279Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443136098Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"session_id":"cc32d12c-7460-4617-a30b-7a8b5817b7e8"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443145176Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902} {"msg":"Activated sub_volume 39531026-a26d-49db-8499-cd766714f019","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443153026Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443161155Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.443480257Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"task":"crutest"} 01/30 Flush 02/30 Flush 03/30 Write block 554 len 32768 data: 23 25 19 14 15 15 12 13 04/30 Read block 271 len 20480 05/30 Flush 06/30 Write block 458 len 24576 data: 13 12 12 9 9 9 07/30 Write block 525 len 24576 data: 13 9 9 10 11 15 08/30 Write block 197 len 24576 data: 12 14 8 8 9 13 09/30 Read block 206 len 4096 10/30 Read block 72 len 28672 11/30 Read block 352 len 8192 12/30 Read block 15 len 12288 13/30 Write block 164 len 24576 data: 12 14 16 13 14 14 14/30 Read block 367 len 20480 15/30 Write block 233 len 4096 data: 12 16/30 Write block 35 len 40960 data: 13 13 13 15 13 15 13 12 14 15 17/30 Write block 486 len 32768 data: 13 13 14 13 15 15 18 18 18/30 Read block 183 len 36864 19/30 Read block 422 len 36864 20/30 Read block 451 len 8192 21/30 Flush 22/30 Write block 17 len 24576 data: 13 13 14 13 13 14 23/30 Flush 24/30 Write block 131 len 32768 data: 16 15 12 11 13 12 11 12 25/30 Write block 424 len 36864 data: 9 7 12 13 14 13 14 15 16 26/30 Read block 384 len 32768 27/30 Write block 531 len 40960 data: 15 16 18 17 18 16 13 16 14 14 28/30 Write block 173 len 24576 data: 15 10 11 12 11 16 29/30 Write block 65 len 28672 data: 14 13 12 14 14 13 11 30/30 Write block 417 len 4096 data: 16 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 8 Done Done Sent false 1003 Acked Read 5 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 6 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 30 36 Done 27 27 0 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:46:01.452636339Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:01.452841835Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1902,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 186 339 339 328 F F F <--- 1 020-039 190 190 186 339 339 330 F F F <--- 2 040-059 190 190 189 338 338 334 F F F <--- 3 060-079 189 189 189 334 334 334 T T F <--- 4 080-099 189 189 189 334 334 334 F F F 5 100-119 189 189 189 334 334 334 F F F 6 120-139 189 189 189 334 334 334 T T F <--- 7 140-159 184 184 184 326 326 326 F F F 8 160-179 190 190 186 338 338 329 T T F <--- 9 180-199 190 190 187 338 338 331 F F F <--- 10 200-219 190 190 186 338 338 328 F F F <--- 11 220-239 190 190 181 338 338 315 F F F <--- 12 240-259 189 189 189 334 334 334 F F F 13 260-279 189 189 189 334 334 334 F F F 14 280-299 187 187 187 331 331 331 F F F 15 300-319 186 186 186 328 328 328 F F F 16 320-339 189 189 189 334 334 334 F F F 17 340-359 189 189 189 334 334 334 F F F 18 360-379 189 189 189 334 334 334 F F F 19 380-399 189 189 189 334 334 334 F F F 20 400-419 189 189 189 334 334 334 T T F <--- 21 420-439 188 188 188 332 332 332 T T F <--- 22 440-459 190 190 187 338 338 331 F F F <--- 23 460-479 190 190 187 338 338 331 F F F <--- 24 480-499 190 190 187 338 338 331 F F F <--- 25 500-519 189 189 189 334 334 334 F F F 26 520-539 190 190 187 338 338 331 T T F <--- 27 540-559 190 190 181 337 337 315 T T F <--- 28 560-579 190 190 189 337 337 334 F F F <--- 29 580-599 178 178 178 309 309 309 F F F Max gen: 190, Max flush: 339 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 339 339 339 F F F 1 020-039 190 190 190 339 339 339 F F F 2 040-059 190 190 190 338 338 338 F F F 3 060-079 191 191 191 340 340 340 F F F 4 080-099 189 189 189 334 334 334 F F F 5 100-119 189 189 189 334 334 334 F F F 6 120-139 191 191 191 340 340 340 F F F 7 140-159 184 184 184 326 326 326 F F F 8 160-179 191 191 191 340 340 340 F F F 9 180-199 190 190 190 338 338 338 F F F 10 200-219 190 190 190 338 338 338 F F F 11 220-239 190 190 190 338 338 338 F F F 12 240-259 189 189 189 334 334 334 F F F 13 260-279 189 189 189 334 334 334 F F F 14 280-299 187 187 187 331 331 331 F F F 15 300-319 186 186 186 328 328 328 F F F 16 320-339 189 189 189 334 334 334 F F F 17 340-359 189 189 189 334 334 334 F F F 18 360-379 189 189 189 334 334 334 F F F 19 380-399 189 189 189 334 334 334 F F F 20 400-419 191 191 191 340 340 340 F F F 21 420-439 191 191 191 340 340 340 F F F 22 440-459 190 190 190 338 338 338 F F F 23 460-479 190 190 190 338 338 338 F F F 24 480-499 190 190 190 338 338 338 F F F 25 500-519 189 189 189 334 334 334 F F F 26 520-539 191 191 191 340 340 340 F F F 27 540-559 191 191 191 340 340 340 F F F 28 560-579 190 190 190 337 337 337 F F F 29 580-599 178 178 178 309 309 309 F F F Max gen: 191, Max flush: 340 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.711771014Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.71986204Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720143597Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720181982Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720193641Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.72020549Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910} {"msg":"Crucible 87034759-6e52-4ebd-be53-89a3070b9d2d has session id: 0f316c7e-247e-4d1a-9e7c-f9e157a51570","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720493606Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"Upstairs opts: Upstairs UUID: 87034759-6e52-4ebd-be53-89a3070b9d2d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720522253Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"Crucible stats registered with UUID: 87034759-6e52-4ebd-be53-89a3070b9d2d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720533241Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:46:05.72054238Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720673135Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720697222Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720707721Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720835206Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720854424Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910} {"msg":"87034759-6e52-4ebd-be53-89a3070b9d2d active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720877771Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.72088716Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"client":"0","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720896999Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"client":"1","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720906588Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"client":"2","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720918246Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.72097243Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.720996407Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"ds_connection connected from Ok(127.0.0.1:35698)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.721027694Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"ds_connection connected from Ok(127.0.0.1:65016)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.721052631Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"ds_connection connected from Ok(127.0.0.1:62771)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.721088367Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.721674098Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"client":"2","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.721720563Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"client":"1","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.721732521Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"client":"0","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.721971973Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722009689Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"[0]R flush_numbers[0..12]: [339, 339, 338, 340, 334, 334, 340, 326, 340, 338, 338, 338]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722070592Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"downstairs","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"[0]R generation[0..12]: [190, 190, 190, 191, 189, 189, 191, 184, 191, 190, 190, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722082291Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"downstairs","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722093599Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"downstairs","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"[1]R flush_numbers[0..12]: [339, 339, 338, 340, 334, 334, 340, 326, 340, 338, 338, 338]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722106408Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"downstairs","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"[1]R generation[0..12]: [190, 190, 190, 191, 189, 189, 191, 184, 191, 190, 190, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722117107Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"downstairs","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722127965Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"downstairs","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"[2]R flush_numbers[0..12]: [339, 339, 338, 340, 334, 334, 340, 326, 340, 338, 338, 338]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722138624Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"downstairs","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"[2]R generation[0..12]: [190, 190, 190, 191, 189, 189, 191, 184, 191, 190, 190, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722149253Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"downstairs","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722160361Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"downstairs","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.72216971Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"downstairs","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722179719Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"downstairs","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722189178Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"downstairs","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722198727Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"":"downstairs","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722208546Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722217415Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"client":"0","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722227104Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"client":"1","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722236813Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"client":"2","":"downstairs","session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722246711Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.72225713Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"87034759-6e52-4ebd-be53-89a3070b9d2d is now active with session: 0f316c7e-247e-4d1a-9e7c-f9e157a51570","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722267139Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722280827Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"session_id":"0f316c7e-247e-4d1a-9e7c-f9e157a51570"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722290106Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910} {"msg":"Activated sub_volume 87034759-6e52-4ebd-be53-89a3070b9d2d","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722298225Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722306444Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.722533168Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"task":"crutest"} 01/30 Write block 265 len 4096 data: 12 02/30 Read block 537 len 32768 03/30 Read block 233 len 28672 04/30 Read block 406 len 28672 05/30 Write block 42 len 12288 data: 13 15 16 06/30 Write block 531 len 4096 data: 16 07/30 Write block 114 len 40960 data: 9 12 16 17 18 18 17 17 17 17 08/30 Read block 244 len 8192 09/30 Read block 222 len 12288 10/30 Flush 11/30 Write block 534 len 24576 data: 18 19 17 14 17 15 12/30 Write block 155 len 8192 data: 17 17 13/30 Read block 392 len 12288 14/30 Write block 268 len 32768 data: 21 24 23 20 19 20 20 19 15/30 Write block 248 len 20480 data: 19 16 18 17 15 16/30 Write block 267 len 16384 data: 19 22 25 24 17/30 Write block 398 len 28672 data: 20 17 18 14 10 11 12 18/30 Read block 361 len 8192 19/30 Read block 67 len 36864 20/30 Write block 334 len 40960 data: 16 17 15 17 13 14 13 10 9 9 21/30 Flush 22/30 Write block 205 len 16384 data: 13 13 11 15 23/30 Read block 161 len 24576 24/30 Write block 171 len 4096 data: 15 25/30 Write block 33 len 4096 data: 20 26/30 Read block 217 len 20480 27/30 Read block 168 len 4096 28/30 Read block 352 len 36864 29/30 Write block 400 len 28672 data: 19 15 11 12 13 9 10 30/30 Write block 379 len 8192 data: 11 12 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Sent Done false 1001 Acked Read 8 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Write 8 Done Sent Done false 1014 Acked Write 5 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Write 10 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Write 1 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:46:05.730164917Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:05.730350385Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1910,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 339 339 339 F F F 1 020-039 190 190 190 339 339 339 T F T <--- 2 040-059 192 190 192 341 338 341 F F F <--- 3 060-079 191 191 191 340 340 340 F F F 4 080-099 189 189 189 334 334 334 F F F 5 100-119 192 189 192 341 334 341 F F F <--- 6 120-139 192 191 192 341 340 341 F F F <--- 7 140-159 192 184 192 342 326 342 F F F <--- 8 160-179 191 191 191 340 340 340 T F T <--- 9 180-199 190 190 190 338 338 338 F F F 10 200-219 190 190 190 338 338 338 T F T <--- 11 220-239 190 190 190 338 338 338 F F F 12 240-259 192 189 192 342 334 342 F F F <--- 13 260-279 192 189 192 342 334 342 F F F <--- 14 280-299 187 187 187 331 331 331 F F F 15 300-319 186 186 186 328 328 328 F F F 16 320-339 192 189 192 342 334 342 F F F <--- 17 340-359 192 189 192 342 334 342 F F F <--- 18 360-379 189 189 189 334 334 334 T F T <--- 19 380-399 192 189 192 342 334 342 T F T <--- 20 400-419 192 191 192 342 340 342 T F T <--- 21 420-439 191 191 191 340 340 340 F F F 22 440-459 190 190 190 338 338 338 F F F 23 460-479 190 190 190 338 338 338 F F F 24 480-499 190 190 190 338 338 338 F F F 25 500-519 189 189 189 334 334 334 F F F 26 520-539 192 191 192 342 340 342 F F F <--- 27 540-559 191 191 191 340 340 340 F F F 28 560-579 190 190 190 337 337 337 F F F 29 580-599 178 178 178 309 309 309 F F F Max gen: 192, Max flush: 342 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 190 190 339 339 339 F F F 1 020-039 193 193 193 343 343 343 F F F 2 040-059 192 192 192 341 341 341 F F F 3 060-079 191 191 191 340 340 340 F F F 4 080-099 189 189 189 334 334 334 F F F 5 100-119 192 192 192 341 341 341 F F F 6 120-139 192 192 192 341 341 341 F F F 7 140-159 192 192 192 342 342 342 F F F 8 160-179 193 193 193 343 343 343 F F F 9 180-199 190 190 190 338 338 338 F F F 10 200-219 193 193 193 343 343 343 F F F 11 220-239 190 190 190 338 338 338 F F F 12 240-259 192 192 192 342 342 342 F F F 13 260-279 192 192 192 342 342 342 F F F 14 280-299 187 187 187 331 331 331 F F F 15 300-319 186 186 186 328 328 328 F F F 16 320-339 192 192 192 342 342 342 F F F 17 340-359 192 192 192 342 342 342 F F F 18 360-379 193 193 193 343 343 343 F F F 19 380-399 193 193 193 343 343 343 F F F 20 400-419 193 193 193 343 343 343 F F F 21 420-439 191 191 191 340 340 340 F F F 22 440-459 190 190 190 338 338 338 F F F 23 460-479 190 190 190 338 338 338 F F F 24 480-499 190 190 190 338 338 338 F F F 25 500-519 189 189 189 334 334 334 F F F 26 520-539 192 192 192 342 342 342 F F F 27 540-559 191 191 191 340 340 340 F F F 28 560-579 190 190 190 337 337 337 F F F 29 580-599 178 178 178 309 309 309 F F F Max gen: 193, Max flush: 343 Begin loop 97 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:09.985041379Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:46:09.992240859Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:09.992296852Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:09.999204716Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:09.999459986Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:09.999489613Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:09.999498112Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:09.99950953Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918} {"msg":"Crucible aab2a4ae-c8f2-427f-8fa3-f03a559b98e3 has session id: fe7da435-7fca-4366-8a33-84ca84ba22fe","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:09.999839782Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"Upstairs opts: Upstairs UUID: aab2a4ae-c8f2-427f-8fa3-f03a559b98e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:09.999890416Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"Crucible stats registered with UUID: aab2a4ae-c8f2-427f-8fa3-f03a559b98e3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:09.999901115Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:46:09.999910274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.000045478Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.000060336Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.000070455Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.000350252Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.00036865Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918} {"msg":"aab2a4ae-c8f2-427f-8fa3-f03a559b98e3 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.000395147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.000404756Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"client":"0","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.000414705Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"client":"1","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.000424454Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"client":"2","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.000435362Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.000445351Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.000476528Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:62799)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.0005412Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:34100)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.000566687Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:42363)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.000578616Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001274084Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"client":"1","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001318189Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"client":"2","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001354615Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"client":"0","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001614685Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001638872Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"[0]R flush_numbers[0..12]: [339, 343, 341, 340, 334, 341, 341, 342, 343, 338, 343, 338]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001692876Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"downstairs","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"[0]R generation[0..12]: [190, 193, 192, 191, 189, 192, 192, 192, 193, 190, 193, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001704834Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"downstairs","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001716033Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"downstairs","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"[1]R flush_numbers[0..12]: [339, 343, 341, 340, 334, 341, 341, 342, 343, 338, 343, 338]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001726822Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"downstairs","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"[1]R generation[0..12]: [190, 193, 192, 191, 189, 192, 192, 192, 193, 190, 193, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.00173735Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"downstairs","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001748659Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"downstairs","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"[2]R flush_numbers[0..12]: [339, 343, 341, 340, 334, 341, 341, 342, 343, 338, 343, 338]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001762687Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"downstairs","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"[2]R generation[0..12]: [190, 193, 192, 191, 189, 192, 192, 192, 193, 190, 193, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001773266Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"downstairs","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001784105Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"downstairs","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001793544Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"downstairs","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001803253Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"downstairs","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"Next flush: 344","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001812732Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"downstairs","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.00182201Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"":"downstairs","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001831809Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001840508Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"client":"0","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001850407Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"client":"1","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001859986Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"client":"2","":"downstairs","session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001869975Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001879174Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"aab2a4ae-c8f2-427f-8fa3-f03a559b98e3 is now active with session: fe7da435-7fca-4366-8a33-84ca84ba22fe","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001889143Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001897962Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"session_id":"fe7da435-7fca-4366-8a33-84ca84ba22fe"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001907201Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918} {"msg":"Activated sub_volume aab2a4ae-c8f2-427f-8fa3-f03a559b98e3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001917049Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.001925198Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.002211195Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"task":"crutest"} 01/30 Write block 37 len 40960 data: 14 16 14 16 14 14 16 17 16 17 02/30 Read block 198 len 4096 03/30 Write block 391 len 36864 data: 15 15 15 16 15 17 17 21 18 04/30 Write block 196 len 20480 data: 12 13 15 9 9 05/30 Read block 274 len 4096 06/30 Read block 348 len 40960 07/30 Read block 422 len 28672 08/30 Read block 294 len 20480 09/30 Read block 0 len 20480 10/30 Write block 558 len 40960 data: 16 16 13 14 14 15 17 14 12 14 11/30 Read block 256 len 4096 12/30 Read block 92 len 8192 13/30 Write block 64 len 32768 data: 14 15 14 13 15 15 14 12 14/30 Write block 5 len 12288 data: 17 18 18 15/30 Read block 13 len 16384 16/30 Write block 503 len 12288 data: 16 17 18 17/30 Write block 491 len 12288 data: 16 19 19 18/30 Flush 19/30 Write block 506 len 16384 data: 17 17 12 13 20/30 Write block 154 len 12288 data: 17 18 18 21/30 Write block 352 len 28672 data: 20 22 23 19 16 13 14 22/30 Write block 277 len 40960 data: 18 17 17 14 12 13 11 15 13 12 23/30 Read block 197 len 32768 24/30 Read block 54 len 4096 25/30 Write block 264 len 4096 data: 14 26/30 Write block 12 len 16384 data: 15 14 12 10 27/30 Read block 582 len 8192 28/30 Write block 174 len 40960 data: 11 12 13 12 17 16 17 16 14 15 29/30 Write block 129 len 20480 data: 19 18 17 16 13 30/30 Read block 240 len 16384 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Sent Done Done false 1001 Acked Read 1 Sent Done Done false 1002 Acked Write 9 Sent Done Done false 1003 Acked Write 5 Sent Done Done false 1004 Acked Read 1 Sent Done Done false 1005 Acked Read 10 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Read 5 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Write 5 Sent Done Sent false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 2 32 Done 0 30 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:46:10.01116182Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:10.011346929Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1918,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 190 194 194 339 344 344 F T T <--- 1 020-039 193 194 194 343 344 344 F F F <--- 2 040-059 192 194 194 341 344 344 F F F <--- 3 060-079 191 194 194 340 344 344 F F F <--- 4 080-099 189 189 189 334 334 334 F F F 5 100-119 192 192 192 341 341 341 F F F 6 120-139 192 192 192 341 341 341 F T T <--- 7 140-159 192 192 192 342 342 342 F T T <--- 8 160-179 193 193 193 343 343 343 F T T <--- 9 180-199 190 194 194 338 344 344 F T T <--- 10 200-219 193 194 194 343 344 344 F F F <--- 11 220-239 190 190 190 338 338 338 F F F 12 240-259 192 192 192 342 342 342 F F F 13 260-279 192 192 192 342 342 342 F T T <--- 14 280-299 187 187 187 331 331 331 F T T <--- 15 300-319 186 186 186 328 328 328 F F F 16 320-339 192 192 192 342 342 342 F F F 17 340-359 192 192 192 342 342 342 F T T <--- 18 360-379 193 193 193 343 343 343 F F F 19 380-399 193 194 194 343 344 344 F F F <--- 20 400-419 193 193 193 343 343 343 F F F 21 420-439 191 191 191 340 340 340 F F F 22 440-459 190 190 190 338 338 338 F F F 23 460-479 190 190 190 338 338 338 F F F 24 480-499 190 194 194 338 344 344 F F F <--- 25 500-519 189 194 194 334 344 344 F T T <--- 26 520-539 192 192 192 342 342 342 F F F 27 540-559 191 194 194 340 344 344 F F F <--- 28 560-579 190 194 194 337 344 344 F F F <--- 29 580-599 178 178 178 309 309 309 F F F Max gen: 194, Max flush: 344 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 345 345 345 F F F 1 020-039 194 194 194 344 344 344 F F F 2 040-059 194 194 194 344 344 344 F F F 3 060-079 194 194 194 344 344 344 F F F 4 080-099 189 189 189 334 334 334 F F F 5 100-119 192 192 192 341 341 341 F F F 6 120-139 195 195 195 345 345 345 F F F 7 140-159 195 195 195 345 345 345 F F F 8 160-179 195 195 195 345 345 345 F F F 9 180-199 195 195 195 345 345 345 F F F 10 200-219 194 194 194 344 344 344 F F F 11 220-239 190 190 190 338 338 338 F F F 12 240-259 192 192 192 342 342 342 F F F 13 260-279 195 195 195 345 345 345 F F F 14 280-299 195 195 195 345 345 345 F F F 15 300-319 186 186 186 328 328 328 F F F 16 320-339 192 192 192 342 342 342 F F F 17 340-359 195 195 195 345 345 345 F F F 18 360-379 193 193 193 343 343 343 F F F 19 380-399 194 194 194 344 344 344 F F F 20 400-419 193 193 193 343 343 343 F F F 21 420-439 191 191 191 340 340 340 F F F 22 440-459 190 190 190 338 338 338 F F F 23 460-479 190 190 190 338 338 338 F F F 24 480-499 194 194 194 344 344 344 F F F 25 500-519 195 195 195 345 345 345 F F F 26 520-539 192 192 192 342 342 342 F F F 27 540-559 194 194 194 344 344 344 F F F 28 560-579 194 194 194 344 344 344 F F F 29 580-599 178 178 178 309 309 309 F F F Max gen: 195, Max flush: 345 Begin loop 98 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.279673751Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.287975332Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.2882484Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.288272687Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.288281506Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.288292995Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926} {"msg":"Crucible 0bb14e2f-f935-447b-bd10-351229549963 has session id: 6d8ab836-4512-4fb5-b5f6-ffe300d22b02","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.288647184Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"Upstairs opts: Upstairs UUID: 0bb14e2f-f935-447b-bd10-351229549963, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.288695848Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"Crucible stats registered with UUID: 0bb14e2f-f935-447b-bd10-351229549963","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.288706297Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:46:14.288715486Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.288855839Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.288870448Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.288880447Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.28902517Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.289040568Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926} {"msg":"0bb14e2f-f935-447b-bd10-351229549963 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.289074204Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.289083823Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"client":"0","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.289093682Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"client":"1","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.289103231Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"client":"2","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.289112809Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.289122748Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.289132627Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"ds_connection connected from Ok(127.0.0.1:59752)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.28919424Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"ds_connection connected from Ok(127.0.0.1:54151)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.289222007Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"ds_connection connected from Ok(127.0.0.1:41267)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.289233725Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.289841414Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"client":"1","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.2898825Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"client":"0","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.289907977Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"client":"2","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290178995Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290201992Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"[0]R flush_numbers[0..12]: [345, 344, 344, 344, 334, 341, 345, 345, 345, 345, 344, 338]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290258816Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"downstairs","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"[0]R generation[0..12]: [195, 194, 194, 194, 189, 192, 195, 195, 195, 195, 194, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290270714Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"downstairs","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290281693Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"downstairs","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"[1]R flush_numbers[0..12]: [345, 344, 344, 344, 334, 341, 345, 345, 345, 345, 344, 338]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290292352Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"downstairs","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"[1]R generation[0..12]: [195, 194, 194, 194, 189, 192, 195, 195, 195, 195, 194, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290302971Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"downstairs","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290313819Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"downstairs","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"[2]R flush_numbers[0..12]: [345, 344, 344, 344, 334, 341, 345, 345, 345, 345, 344, 338]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290324458Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"downstairs","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"[2]R generation[0..12]: [195, 194, 194, 194, 189, 192, 195, 195, 195, 195, 194, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290335147Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"downstairs","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290345925Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"downstairs","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290355384Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"downstairs","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290367423Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"downstairs","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"Next flush: 346","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290376912Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"downstairs","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290386481Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"":"downstairs","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.29039637Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290405389Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"client":"0","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290415177Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"client":"1","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290424986Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"client":"2","":"downstairs","session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290434775Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290443934Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"0bb14e2f-f935-447b-bd10-351229549963 is now active with session: 6d8ab836-4512-4fb5-b5f6-ffe300d22b02","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290453853Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290462602Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"session_id":"6d8ab836-4512-4fb5-b5f6-ffe300d22b02"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290471781Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926} {"msg":"Activated sub_volume 0bb14e2f-f935-447b-bd10-351229549963","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.29047962Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290487919Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.290805942Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"task":"crutest"} 01/30 Write block 257 len 4096 data: 14 02/30 Read block 491 len 28672 03/30 Read block 278 len 12288 04/30 Read block 317 len 28672 05/30 Read block 240 len 8192 06/30 Read block 565 len 20480 07/30 Flush 08/30 Write block 565 len 28672 data: 15 13 15 16 11 14 16 09/30 Flush 10/30 Write block 244 len 8192 data: 21 18 11/30 Read block 427 len 40960 12/30 Read block 419 len 28672 13/30 Flush 14/30 Write block 551 len 12288 data: 24 24 24 15/30 Read block 66 len 40960 16/30 Write block 162 len 32768 data: 9 11 13 15 17 14 15 15 17/30 Read block 401 len 12288 18/30 Read block 259 len 24576 19/30 Read block 97 len 8192 20/30 Flush 21/30 Write block 305 len 40960 data: 21 19 20 17 17 17 11 11 11 8 22/30 Read block 561 len 28672 23/30 Read block 470 len 32768 24/30 Write block 375 len 32768 data: 11 14 13 11 12 13 11 14 25/30 Write block 278 len 40960 data: 18 18 15 13 14 12 16 14 13 15 26/30 Read block 120 len 32768 27/30 Write block 129 len 20480 data: 20 19 18 17 14 28/30 Write block 414 len 36864 data: 15 14 16 17 14 15 14 15 12 29/30 Read block 458 len 40960 30/30 Read block 137 len 32768 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 7 Done Done Sent false 1002 Acked Read 3 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Read 2 Done Done Sent false 1005 Acked Read 5 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 3 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 10 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 29 30 Done 29 30 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:46:14.300139382Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:14.300324861Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1926,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 345 345 345 F F F 1 020-039 194 194 194 344 344 344 F F F 2 040-059 194 194 194 344 344 344 F F F 3 060-079 194 194 194 344 344 344 F F F 4 080-099 189 189 189 334 334 334 F F F 5 100-119 192 192 192 341 341 341 F F F 6 120-139 195 195 195 345 345 345 T T F <--- 7 140-159 195 195 195 345 345 345 F F F 8 160-179 196 196 195 349 349 345 F F F <--- 9 180-199 195 195 195 345 345 345 F F F 10 200-219 194 194 194 344 344 344 F F F 11 220-239 190 190 190 338 338 338 F F F 12 240-259 196 196 192 348 348 342 F F T <--- 13 260-279 195 195 195 345 345 345 T T F <--- 14 280-299 195 195 195 345 345 345 T T F <--- 15 300-319 186 186 186 328 328 328 T T F <--- 16 320-339 192 192 192 342 342 342 F F F 17 340-359 195 195 195 345 345 345 F F F 18 360-379 193 193 193 343 343 343 T T F <--- 19 380-399 194 194 194 344 344 344 T T F <--- 20 400-419 193 193 193 343 343 343 T T F <--- 21 420-439 191 191 191 340 340 340 T T F <--- 22 440-459 190 190 190 338 338 338 F F F 23 460-479 190 190 190 338 338 338 F F F 24 480-499 194 194 194 344 344 344 F F F 25 500-519 195 195 195 345 345 345 F F F 26 520-539 192 192 192 342 342 342 F F F 27 540-559 196 196 194 349 349 344 F F F <--- 28 560-579 196 196 194 347 347 344 F F F <--- 29 580-599 178 178 178 309 309 309 F F F Max gen: 196, Max flush: 349 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 345 345 345 F F F 1 020-039 194 194 194 344 344 344 F F F 2 040-059 194 194 194 344 344 344 F F F 3 060-079 194 194 194 344 344 344 F F F 4 080-099 189 189 189 334 334 334 F F F 5 100-119 192 192 192 341 341 341 F F F 6 120-139 197 197 197 350 350 350 F F F 7 140-159 195 195 195 345 345 345 F F F 8 160-179 196 196 196 349 349 349 F F F 9 180-199 195 195 195 345 345 345 F F F 10 200-219 194 194 194 344 344 344 F F F 11 220-239 190 190 190 338 338 338 F F F 12 240-259 196 196 196 348 348 348 F F F 13 260-279 197 197 197 350 350 350 F F F 14 280-299 197 197 197 350 350 350 F F F 15 300-319 197 197 197 350 350 350 F F F 16 320-339 192 192 192 342 342 342 F F F 17 340-359 195 195 195 345 345 345 F F F 18 360-379 197 197 197 350 350 350 F F F 19 380-399 197 197 197 350 350 350 F F F 20 400-419 197 197 197 350 350 350 F F F 21 420-439 197 197 197 350 350 350 F F F 22 440-459 190 190 190 338 338 338 F F F 23 460-479 190 190 190 338 338 338 F F F 24 480-499 194 194 194 344 344 344 F F F 25 500-519 195 195 195 345 345 345 F F F 26 520-539 192 192 192 342 342 342 F F F 27 540-559 196 196 196 349 349 349 F F F 28 560-579 196 196 196 347 347 347 F F F 29 580-599 178 178 178 309 309 309 F F F Max gen: 197, Max flush: 350 Begin loop 99 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.552720373Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.560769143Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561083087Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"d44fb619fd6f6ba74fbd67c6975e3d16808321ca\",\n git_commit_timestamp: \"2026-09-16T04:23:02.000000000Z\",\n git_branch: \"renovate/crossterm-0.x\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561117083Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561126332Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.56113794Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934} {"msg":"Crucible 0387e824-403b-4e7a-8975-c1c03cb6ea29 has session id: 0a19a4e3-f42c-476d-acfc-4d14b8302c0e","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561432316Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"Upstairs opts: Upstairs UUID: 0387e824-403b-4e7a-8975-c1c03cb6ea29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561460143Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"Crucible stats registered with UUID: 0387e824-403b-4e7a-8975-c1c03cb6ea29","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561471252Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:46:18.56148053Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561605406Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561621464Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561631533Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561756188Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561772926Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934} {"msg":"0387e824-403b-4e7a-8975-c1c03cb6ea29 active request set","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561811702Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561821051Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"client":"0","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561832779Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"client":"1","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561842508Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"client":"2","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561851937Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561861766Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561871695Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56718)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561922609Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:54693)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.561933688Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44509)","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.562002959Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.562539247Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"client":"0","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.562565274Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"client":"1","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.562650604Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"client":"2","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.56285881Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.562878827Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"[0]R flush_numbers[0..12]: [345, 344, 344, 344, 334, 341, 350, 345, 349, 345, 344, 338]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.56293875Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"downstairs","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"[0]R generation[0..12]: [195, 194, 194, 194, 189, 192, 197, 195, 196, 195, 194, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.562950569Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"downstairs","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.562964087Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"downstairs","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"[1]R flush_numbers[0..12]: [345, 344, 344, 344, 334, 341, 350, 345, 349, 345, 344, 338]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.562974876Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"downstairs","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"[1]R generation[0..12]: [195, 194, 194, 194, 189, 192, 197, 195, 196, 195, 194, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.562985545Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"downstairs","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.562996523Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"downstairs","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"[2]R flush_numbers[0..12]: [345, 344, 344, 344, 334, 341, 350, 345, 349, 345, 344, 338]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563007082Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"downstairs","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"[2]R generation[0..12]: [195, 194, 194, 194, 189, 192, 197, 195, 196, 195, 194, 190]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563017631Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"downstairs","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.5630287Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"downstairs","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563038639Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"downstairs","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563048397Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"downstairs","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"Next flush: 351","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563057746Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"downstairs","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563067275Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"":"downstairs","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563077404Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563086273Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"client":"0","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563096142Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"client":"1","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563107221Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"client":"2","":"downstairs","session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563117349Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563126608Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"0387e824-403b-4e7a-8975-c1c03cb6ea29 is now active with session: 0a19a4e3-f42c-476d-acfc-4d14b8302c0e","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563136537Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563145316Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"session_id":"0a19a4e3-f42c-476d-acfc-4d14b8302c0e"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563154395Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934} {"msg":"Activated sub_volume 0387e824-403b-4e7a-8975-c1c03cb6ea29","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563162444Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-09-16T04:46:18.563170553Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.563430483Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"task":"crutest"} 01/30 Write block 588 len 4096 data: 12 02/30 Write block 262 len 12288 data: 13 15 15 03/30 Write block 30 len 12288 data: 23 25 23 04/30 Read block 344 len 36864 05/30 Write block 422 len 40960 data: 13 11 10 8 13 14 15 14 15 16 06/30 Flush 07/30 Write block 360 len 32768 data: 15 13 12 13 12 10 11 12 08/30 Write block 119 len 12288 data: 19 18 18 09/30 Write block 449 len 8192 data: 15 16 10/30 Write block 552 len 28672 data: 25 25 24 26 20 15 17 11/30 Read block 3 len 16384 12/30 Read block 491 len 20480 13/30 Flush 14/30 Write block 91 len 36864 data: 17 20 20 20 18 16 10 10 9 15/30 Flush 16/30 Write block 200 len 32768 data: 10 10 14 14 14 14 14 12 17/30 Read block 555 len 28672 18/30 Read block 430 len 4096 19/30 Write block 38 len 28672 data: 17 15 17 15 15 17 18 20/30 Read block 103 len 4096 21/30 Read block 201 len 4096 22/30 Read block 387 len 36864 23/30 Write block 16 len 36864 data: 12 14 14 15 14 14 15 14 17 24/30 Flush 25/30 Write block 240 len 36864 data: 17 20 21 19 22 19 18 18 20 26/30 Read block 480 len 4096 27/30 Write block 60 len 8192 data: 17 18 28/30 Read block 534 len 12288 29/30 Write block 495 len 24576 data: 18 17 14 13 15 16 30/30 Read block 153 len 4096 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 8 Done Done Done false 1007 Acked Write 3 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Write 7 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 8 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Write 7 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 21 21 Done 24 24 3 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-09-16T04:46:18.572230456Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-09-16T04:46:18.572409165Z","hostname":"w-01M2M83DXJQVMX3E5Q7KW28XKF","pid":1934,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 195 354 354 345 F F F <--- 1 020-039 198 198 198 354 354 351 F F F <--- 2 040-059 198 198 194 354 354 344 F F F <--- 3 060-079 194 194 194 344 344 344 T T F <--- 4 080-099 198 198 189 353 353 334 F F F <--- 5 100-119 198 198 192 352 352 341 F F T <--- 6 120-139 198 198 197 352 352 350 F F T <--- 7 140-159 195 195 195 345 345 345 F F F 8 160-179 196 196 196 349 349 349 F F F 9 180-199 195 195 195 345 345 345 F F F 10 200-219 198 198 194 354 354 344 F F F <--- 11 220-239 190 190 190 338 338 338 F F F 12 240-259 196 196 196 348 348 348 T T F <--- 13 260-279 198 198 198 351 351 351 F F F 14 280-299 197 197 197 350 350 350 F F F 15 300-319 197 197 197 350 350 350 F F F 16 320-339 192 192 192 342 342 342 F F F 17 340-359 195 195 195 345 345 345 F F F 18 360-379 198 198 197 352 352 350 F F T <--- 19 380-399 197 197 197 350 350 350 F F F 20 400-419 197 197 197 350 350 350 F F F 21 420-439 198 198 198 351 351 351 F F F 22 440-459 198 198 190 352 352 338 F F T <--- 23 460-479 190 190 190 338 338 338 F F F 24 480-499 194 194 194 344 344 344 T T F <--- 25 500-519 195 195 195 345 345 345 T T F <--- 26 520-539 192 192 192 342 342 342 F F F 27 540-559 198 198 196 352 352 349 F F F <--- 28 560-579 196 196 196 347 347 347 F F F 29 580-599 198 198 198 351 351 351 F F F Max gen: 198, Max flush: 354 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 354 354 354 F F F 1 020-039 198 198 198 354 354 354 F F F 2 040-059 198 198 198 354 354 354 F F F 3 060-079 199 199 199 355 355 355 F F F 4 080-099 198 198 198 353 353 353 F F F 5 100-119 198 198 198 352 352 352 F F F 6 120-139 198 198 198 352 352 352 F F F 7 140-159 195 195 195 345 345 345 F F F 8 160-179 196 196 196 349 349 349 F F F 9 180-199 195 195 195 345 345 345 F F F 10 200-219 198 198 198 354 354 354 F F F 11 220-239 190 190 190 338 338 338 F F F 12 240-259 199 199 199 355 355 355 F F F 13 260-279 198 198 198 351 351 351 F F F 14 280-299 197 197 197 350 350 350 F F F 15 300-319 197 197 197 350 350 350 F F F 16 320-339 192 192 192 342 342 342 F F F 17 340-359 195 195 195 345 345 345 F F F 18 360-379 198 198 198 352 352 352 F F F 19 380-399 197 197 197 350 350 350 F F F 20 400-419 197 197 197 350 350 350 F F F 21 420-439 198 198 198 351 351 351 F F F 22 440-459 198 198 198 352 352 352 F F F 23 460-479 190 190 190 338 338 338 F F F 24 480-499 199 199 199 355 355 355 F F F 25 500-519 199 199 199 355 355 355 F F F 26 520-539 192 192 192 342 342 342 F F F 27 540-559 198 198 198 352 352 352 F F F 28 560-579 196 196 196 347 347 347 F F F 29 580-599 198 198 198 351 351 351 F F F Max gen: 199, Max flush: 355 7:27 Test duration Test completed real 7:27.626751477 user 25.334308111 sys 14.255381192 trap 0.018142814 tflt 0.059481051 dflt 0.072236903 kflt 0.003636820 lock 11:51:36.199885871 slp 36:06.532023243 lat 2.314102886 stop 0.019409956