Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.20392989 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.013492262 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.012471311 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1134 1135 1136 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.944366276Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.961459433Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.963634895Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.963687051Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.963759286Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.963809203Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139} {"msg":"Crucible d39c094b-8351-441e-9768-548b92dde139 has session id: 9805cef5-504e-4d79-ac6a-a26f4be3644b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.964773937Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"Upstairs opts: Upstairs UUID: d39c094b-8351-441e-9768-548b92dde139, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.9648817Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"Crucible stats registered with UUID: d39c094b-8351-441e-9768-548b92dde139","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.964953045Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:38:49.96502043Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.96620008Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.96650045Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.966514889Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.966525718Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"task":"crutest"} {"msg":"d39c094b-8351-441e-9768-548b92dde139 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.966601503Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.966611712Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"client":"0","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.966622062Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.966700656Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"client":"1","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.966711335Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"client":"2","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.966753903Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.966765102Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.966775351Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50189)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.967131317Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"io task","client":"0","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45826)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.967152115Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"io task","client":"1","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49829)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.967163085Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"io task","client":"2","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.968931034Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"client":"1","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.968961742Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"client":"2","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.968976021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"client":"0","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969575771Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.96958633Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.96972531Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"downstairs","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.96973682Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"downstairs","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969748089Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"downstairs","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969758748Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"downstairs","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969769337Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"downstairs","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969780547Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"downstairs","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969791026Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"downstairs","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969801475Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"downstairs","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969812674Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"downstairs","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969822414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"downstairs","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969832393Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"downstairs","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969841942Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"downstairs","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969862421Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"":"downstairs","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.96987252Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.9698817Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"client":"0","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969891939Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"client":"1","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969902248Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"client":"2","":"downstairs","session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969912598Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969922177Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"d39c094b-8351-441e-9768-548b92dde139 is now active with session: 9805cef5-504e-4d79-ac6a-a26f4be3644b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969932496Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.969942046Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.970226206Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"session_id":"9805cef5-504e-4d79-ac6a-a26f4be3644b"} {"msg":"Activated sub_volume d39c094b-8351-441e-9768-548b92dde139","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.970293242Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.970302321Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Fill test","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:49.970494438Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"task":"crutest"} Read and Verify all blocks (0..600 range:false) {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 ,"States:time":" Active2026-07-28T04:38:50.008865317Z Active" Active ,"hostname":"JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"task" 1004 Acked Read 88:" Donecrutest" Done Done false } 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false {"msg":" STATES DS:0 DS:1 DS:2 TOTAL CLIENT: Tests done. All submitted work has been ACK'd" Sent ,"v": 0 0 0 , 0 " 0 name":" Done 3 crucible" 3 3 9 ," Skipped 0 level" 0 : 0 30 0 Error 0 0 0 0 ,"time":"Last Flush: Some(JobId(1003)) 2026-07-28T04:38:50.008936023Z"Some(JobId(1003)) ,Some(JobId(1003)) " hostname"Downstairs last five completed::"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139 1003, 1002" 1001 1000task ":"crutest"} Upstairs last five completed: 1006 1005 1004 1003 1002 {"msg":"CLIENT: Up:0 ds:3 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:50.008993369Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"task":"crutest"} ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 {"msg":"CLIENT: Up:0 ds:0 act:3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.009975076Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"task":"crutest"} {"msg":"CLIENT: All crucible jobs finished, exiting program","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.010019623Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1139,"task":"crutest"} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.035245356Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:38:54.042454456Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.042521451Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.049686654Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.049935757Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.049959535Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.049968685Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.049980804Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141} {"msg":"Crucible 9ce1f78f-d57c-4c9f-8647-2bd46224b5b7 has session id: de79e7bc-2f11-411f-ae2f-7859beab7248","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050372247Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"Upstairs opts: Upstairs UUID: 9ce1f78f-d57c-4c9f-8647-2bd46224b5b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050402465Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"Crucible stats registered with UUID: 9ce1f78f-d57c-4c9f-8647-2bd46224b5b7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050416144Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:38:54.050425884Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050592972Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.05062501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050637009Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050737062Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050754351Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141} {"msg":"9ce1f78f-d57c-4c9f-8647-2bd46224b5b7 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.05077252Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050782009Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"client":"0","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050792139Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"client":"1","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050802208Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"client":"2","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050812297Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050822717Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"ds_connection connected from Ok(127.0.0.1:42079)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050882693Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"ds_connection connected from Ok(127.0.0.1:65047)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050907021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.05091825Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"ds_connection connected from Ok(127.0.0.1:63396)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.050951578Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051494891Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"client":"1","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051554477Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"client":"2","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051575985Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"client":"0","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051825298Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051850347Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051867176Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"downstairs","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051878645Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"downstairs","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051890094Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"downstairs","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051900823Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"downstairs","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051911513Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"downstairs","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051922942Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"downstairs","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051933801Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"downstairs","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.05194447Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"downstairs","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051959009Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"downstairs","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051969089Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"downstairs","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051979468Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"downstairs","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051989407Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"downstairs","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.051999367Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"":"downstairs","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.052009536Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.052018815Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"client":"0","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.052028985Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"client":"1","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.052039084Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"client":"2","":"downstairs","session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.052049523Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.052059192Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"9ce1f78f-d57c-4c9f-8647-2bd46224b5b7 is now active with session: de79e7bc-2f11-411f-ae2f-7859beab7248","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.052069772Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.052078941Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"session_id":"de79e7bc-2f11-411f-ae2f-7859beab7248"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.05208842Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141} {"msg":"Activated sub_volume 9ce1f78f-d57c-4c9f-8647-2bd46224b5b7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.052097Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.052105869Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.052569708Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"task":"crutest"} 01/30 Write block 71 len 12288 data: 2 2 2 02/30 Read block 542 len 8192 03/30 Write block 13 len 36864 data: 2 2 2 2 2 2 2 2 2 04/30 Read block 219 len 28672 05/30 Read block 461 len 28672 06/30 Read block 255 len 8192 07/30 Write block 378 len 24576 data: 2 2 2 2 2 2 08/30 Write block 40 len 36864 data: 2 2 2 2 2 2 2 2 2 09/30 Read block 12 len 24576 10/30 Write block 526 len 40960 data: 2 2 2 2 2 2 2 2 2 2 11/30 Read block 451 len 4096 12/30 Read block 362 len 32768 13/30 Read block 536 len 36864 14/30 Flush 15/30 Write block 457 len 40960 data: 2 2 2 2 2 2 2 2 2 2 16/30 Write block 188 len 12288 data: 2 2 2 17/30 Flush 18/30 Write block 268 len 4096 data: 2 19/30 Read block 304 len 20480 20/30 Flush 21/30 Write block 235 len 8192 data: 2 2 22/30 Read block 467 len 24576 23/30 Read block 502 len 36864 24/30 Write block 108 len 36864 data: 2 2 2 2 2 2 2 2 2 25/30 Read block 294 len 20480 26/30 Write block 391 len 36864 data: 2 2 2 2 2 2 2 2 2 27/30 Read block 510 len 32768 28/30 Read block 299 len 16384 29/30 Write block 191 len 24576 data: 2 2 2 2 2 2 30/30 Write block 457 len 40960 data: 3 3 3 3 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Sent Done Done false 1001 Acked Read 2 Sent Done Done false 1002 Acked Write 9 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Read 2 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Write 9 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:38:54.062410598Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:54.062713118Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1141,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 2 2 1 2 2 F F F <--- 1 020-039 1 2 2 1 2 2 F F F <--- 2 040-059 1 2 2 1 2 2 F F F <--- 3 060-079 1 2 2 1 2 2 F F F <--- 4 080-099 1 1 1 1 1 1 F F F 5 100-119 1 1 1 1 1 1 F T T <--- 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 2 2 1 3 3 F T T <--- 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F T T <--- 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 2 2 1 4 4 F F F <--- 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 2 2 1 2 2 F F F <--- 19 380-399 1 2 2 1 2 2 F T T <--- 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 2 2 1 3 3 F T T <--- 23 460-479 1 2 2 1 3 3 F T T <--- 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 1 2 2 1 2 2 F F F <--- 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 4 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 2 2 2 2 2 2 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 3 3 3 5 5 5 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 3 3 3 5 5 5 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 3 3 3 5 5 5 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 3 3 3 5 5 5 F F F 23 460-479 3 3 3 5 5 5 F F F 24 480-499 1 1 1 1 1 1 F F F 25 500-519 1 1 1 1 1 1 F F F 26 520-539 2 2 2 2 2 2 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 5 Begin loop 2 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.439487016Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.447972899Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.448300077Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.448338694Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.448348323Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.448360503Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149} {"msg":"Crucible 1256da89-3836-4204-9ac8-a50f7e1f8768 has session id: ba5e15a8-8dcc-4be4-988e-5dba83047745","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.448709599Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"Upstairs opts: Upstairs UUID: 1256da89-3836-4204-9ac8-a50f7e1f8768, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.448728478Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"Crucible stats registered with UUID: 1256da89-3836-4204-9ac8-a50f7e1f8768","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.448738627Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:38:58.448751726Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.448937863Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.448969541Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.44898069Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449060685Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449087383Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149} {"msg":"1256da89-3836-4204-9ac8-a50f7e1f8768 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449116021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449125741Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"client":"0","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.44913589Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"client":"1","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449145879Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"client":"2","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449155918Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449173797Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449209825Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"ds_connection connected from Ok(127.0.0.1:49944)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449224784Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"io task","client":"2","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"ds_connection connected from Ok(127.0.0.1:41354)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449241223Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"io task","client":"1","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"ds_connection connected from Ok(127.0.0.1:45290)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449259891Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"io task","client":"0","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449938985Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"client":"1","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449970273Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"client":"2","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.449981922Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"client":"0","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450256694Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450268993Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"[0]R flush_numbers[0..12]: [2, 2, 2, 2, 1, 5, 1, 1, 1, 5, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.45030535Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"downstairs","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"[0]R generation[0..12]: [2, 2, 2, 2, 1, 3, 1, 1, 1, 3, 1, 3]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450317159Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"downstairs","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450328629Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"downstairs","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"[1]R flush_numbers[0..12]: [2, 2, 2, 2, 1, 5, 1, 1, 1, 5, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450339248Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"downstairs","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"[1]R generation[0..12]: [2, 2, 2, 2, 1, 3, 1, 1, 1, 3, 1, 3]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450349757Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"downstairs","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450361046Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"downstairs","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"[2]R flush_numbers[0..12]: [2, 2, 2, 2, 1, 5, 1, 1, 1, 5, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450371586Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"downstairs","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"[2]R generation[0..12]: [2, 2, 2, 2, 1, 3, 1, 1, 1, 3, 1, 3]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450381875Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"downstairs","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450393104Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"downstairs","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450402854Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"downstairs","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450412983Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"downstairs","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450422642Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"downstairs","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450432352Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"":"downstairs","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450442431Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.4504516Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"client":"0","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.45046331Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"client":"1","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450473309Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"client":"2","":"downstairs","session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450483418Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450493027Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"1256da89-3836-4204-9ac8-a50f7e1f8768 is now active with session: ba5e15a8-8dcc-4be4-988e-5dba83047745","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450503397Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450512836Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450520746Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"session_id":"ba5e15a8-8dcc-4be4-988e-5dba83047745"} {"msg":"Activated sub_volume 1256da89-3836-4204-9ac8-a50f7e1f8768","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450530215Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450538514Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.450818745Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"task":"crutest"} 01/30 Write block 502 len 16384 data: 2 2 2 2 02/30 Write block 215 len 32768 data: 2 2 2 2 2 2 2 2 03/30 Write block 208 len 36864 data: 2 2 2 2 2 2 2 3 3 04/30 Read block 426 len 16384 05/30 Flush 06/30 Write block 431 len 12288 data: 2 2 2 07/30 Write block 204 len 32768 data: 2 2 2 2 3 3 3 3 08/30 Read block 363 len 32768 09/30 Read block 437 len 4096 10/30 Read block 144 len 20480 11/30 Write block 485 len 24576 data: 2 2 2 2 2 2 12/30 Write block 537 len 36864 data: 2 2 2 2 2 2 2 2 2 13/30 Read block 8 len 32768 14/30 Write block 452 len 32768 data: 2 2 2 2 2 4 4 4 15/30 Read block 363 len 40960 16/30 Write block 219 len 24576 data: 3 3 3 3 2 2 17/30 Write block 573 len 36864 data: 2 2 2 2 2 2 2 2 2 18/30 Write block 167 len 20480 data: 2 2 2 2 2 19/30 Write block 339 len 8192 data: 2 2 20/30 Read block 420 len 4096 21/30 Write block 141 len 40960 data: 2 2 2 2 2 2 2 2 2 2 22/30 Write block 312 len 4096 data: 2 23/30 Read block 317 len 36864 24/30 Write block 549 len 4096 data: 2 25/30 Read block 561 len 8192 26/30 Read block 588 len 12288 27/30 Write block 93 len 28672 data: 2 2 2 2 2 2 2 28/30 Read block 369 len 4096 29/30 Write block 125 len 40960 data: 2 2 2 2 2 2 2 2 2 2 30/30 Write block 327 len 28672 data: 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 3 Done Done Done false 1006 Acked Write 8 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Write 1 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 24 28 Done 23 23 1 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:38:58.458440827Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:38:58.458735337Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1149,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 2 2 2 2 2 2 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 1 1 1 1 1 1 T T F <--- 5 100-119 3 3 3 5 5 5 F F F 6 120-139 1 1 1 1 1 1 T T F <--- 7 140-159 1 1 1 1 1 1 T T F <--- 8 160-179 1 1 1 1 1 1 T T F <--- 9 180-199 3 3 3 5 5 5 F F F 10 200-219 4 4 4 6 6 6 T T F <--- 11 220-239 4 4 4 6 6 6 T T F <--- 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 T T F <--- 16 320-339 1 1 1 1 1 1 T T F <--- 17 340-359 1 1 1 1 1 1 T T F <--- 18 360-379 2 2 2 2 2 2 F F F 19 380-399 3 3 3 5 5 5 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 T T T 22 440-459 3 3 3 5 5 5 T T F <--- 23 460-479 3 3 3 5 5 5 F F F 24 480-499 1 1 1 1 1 1 T T F <--- 25 500-519 4 4 4 6 6 6 F F F 26 520-539 2 2 2 2 2 2 T T F <--- 27 540-559 1 1 1 1 1 1 T T F <--- 28 560-579 1 1 1 1 1 1 T T F <--- 29 580-599 1 1 1 1 1 1 T T F <--- Max gen: 4, Max flush: 6 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 2 2 2 2 2 2 F F F 2 040-059 2 2 2 2 2 2 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 5 5 5 7 7 7 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 5 5 5 7 7 7 F F F 7 140-159 5 5 5 7 7 7 F F F 8 160-179 5 5 5 7 7 7 F F F 9 180-199 3 3 3 5 5 5 F F F 10 200-219 5 5 5 7 7 7 F F F 11 220-239 5 5 5 7 7 7 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 5 5 5 7 7 7 F F F 17 340-359 5 5 5 7 7 7 F F F 18 360-379 2 2 2 2 2 2 F F F 19 380-399 3 3 3 5 5 5 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 5 5 5 7 7 7 F F F 22 440-459 5 5 5 7 7 7 F F F 23 460-479 3 3 3 5 5 5 F F F 24 480-499 5 5 5 7 7 7 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 5 5 5 7 7 7 F F F 27 540-559 5 5 5 7 7 7 F F F 28 560-579 5 5 5 7 7 7 F F F 29 580-599 5 5 5 7 7 7 F F F Max gen: 5, Max flush: 7 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.705090199Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.713196077Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.713478778Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.713505376Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.713514396Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.713526225Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157} {"msg":"Crucible 0727eb41-4bb2-48fd-8069-9a11a1f73535 has session id: ca825ee2-95b8-4e77-83ae-a9e133aef6d2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.713902139Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"Upstairs opts: Upstairs UUID: 0727eb41-4bb2-48fd-8069-9a11a1f73535, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.713925158Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"Crucible stats registered with UUID: 0727eb41-4bb2-48fd-8069-9a11a1f73535","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.713935557Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:02.713948346Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.714125614Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.714156782Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.714167991Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.714279094Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.714302242Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157} {"msg":"0727eb41-4bb2-48fd-8069-9a11a1f73535 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.714316341Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.714326051Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"client":"0","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.71433666Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"client":"1","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.714347159Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"client":"2","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.714357108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.714371507Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:36749)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.714413255Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"io task","client":"2","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.714441333Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:44699)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.71448516Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"io task","client":"1","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"ds_connection connected from Ok(127.0.0.1:41918)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.714504258Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"io task","client":"0","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715110667Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"client":"1","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715138095Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"client":"0","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.71521686Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"client":"2","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715435775Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715446514Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"[0]R flush_numbers[0..12]: [2, 2, 2, 2, 7, 5, 7, 7, 7, 5, 7, 7]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715519349Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"downstairs","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"[0]R generation[0..12]: [2, 2, 2, 2, 5, 3, 5, 5, 5, 3, 5, 5]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715530779Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"downstairs","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715542198Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"downstairs","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"[1]R flush_numbers[0..12]: [2, 2, 2, 2, 7, 5, 7, 7, 7, 5, 7, 7]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715552737Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"downstairs","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"[1]R generation[0..12]: [2, 2, 2, 2, 5, 3, 5, 5, 5, 3, 5, 5]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715563226Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"downstairs","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715574436Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"downstairs","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"[2]R flush_numbers[0..12]: [2, 2, 2, 2, 7, 5, 7, 7, 7, 5, 7, 7]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715587225Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"downstairs","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"[2]R generation[0..12]: [2, 2, 2, 2, 5, 3, 5, 5, 5, 3, 5, 5]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715597694Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"downstairs","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715608853Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"downstairs","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715618613Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"downstairs","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715628582Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"downstairs","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715638181Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"downstairs","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715647901Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"":"downstairs","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.71565806Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715667309Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"client":"0","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715677519Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"client":"1","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715687818Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"client":"2","":"downstairs","session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715698267Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715707807Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"0727eb41-4bb2-48fd-8069-9a11a1f73535 is now active with session: ca825ee2-95b8-4e77-83ae-a9e133aef6d2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715717936Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715727275Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715735175Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"session_id":"ca825ee2-95b8-4e77-83ae-a9e133aef6d2"} {"msg":"Activated sub_volume 0727eb41-4bb2-48fd-8069-9a11a1f73535","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715746024Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.715754533Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.716081221Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"task":"crutest"} 01/30 Write block 529 len 40960 data: 3 3 3 3 3 3 3 2 3 3 02/30 Write block 432 len 28672 data: 3 3 2 2 2 2 2 03/30 Read block 313 len 32768 04/30 Write block 108 len 16384 data: 3 3 3 3 05/30 Read block 89 len 36864 06/30 Flush 07/30 Write block 375 len 20480 data: 2 2 2 3 3 08/30 Read block 290 len 32768 09/30 Read block 465 len 36864 10/30 Read block 165 len 36864 11/30 Write block 238 len 40960 data: 2 2 2 2 2 2 2 2 2 2 12/30 Write block 549 len 32768 data: 3 2 2 2 2 2 2 2 13/30 Flush 14/30 Write block 429 len 32768 data: 2 2 3 4 4 3 3 3 15/30 Read block 283 len 20480 16/30 Read block 272 len 32768 17/30 Flush 18/30 Write block 394 len 36864 data: 3 3 3 3 3 3 2 2 2 19/30 Write block 38 len 20480 data: 2 2 3 3 3 20/30 Write block 372 len 8192 data: 2 2 21/30 Read block 69 len 16384 22/30 Write block 159 len 36864 data: 2 2 2 2 2 2 2 2 3 23/30 Read block 408 len 28672 24/30 Read block 291 len 28672 25/30 Write block 146 len 28672 data: 3 3 3 3 3 2 2 26/30 Read block 105 len 12288 27/30 Read block 118 len 16384 28/30 Write block 245 len 12288 data: 3 3 3 29/30 Write block 159 len 4096 data: 3 30/30 Write block 357 len 40960 data: 2 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 7 Done Done Sent false 1002 Acked Read 8 Done Done Sent false 1003 Acked Write 4 Done Done Sent false 1004 Acked Read 9 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Write 3 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 29 35 Done 27 27 1 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:02.725252137Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:02.725580095Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1157,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 2 2 2 2 2 2 T T F <--- 2 040-059 2 2 2 2 2 2 T T F <--- 3 060-079 2 2 2 2 2 2 F F F 4 080-099 5 5 5 7 7 7 F F F 5 100-119 6 6 3 8 8 5 F F F <--- 6 120-139 5 5 5 7 7 7 F F F 7 140-159 5 5 5 7 7 7 T T F <--- 8 160-179 5 5 5 7 7 7 T T F <--- 9 180-199 3 3 3 5 5 5 F F F 10 200-219 5 5 5 7 7 7 F F F 11 220-239 6 6 5 9 9 7 F F F <--- 12 240-259 6 6 1 9 9 1 T T F <--- 13 260-279 2 2 2 4 4 4 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 5 5 5 7 7 7 F F F 17 340-359 5 5 5 7 7 7 T T F <--- 18 360-379 6 6 2 9 9 2 T T F <--- 19 380-399 3 3 3 5 5 5 T T F <--- 20 400-419 1 1 1 1 1 1 T T F <--- 21 420-439 6 6 5 10 10 7 F F F <--- 22 440-459 5 5 5 7 7 7 F F F 23 460-479 3 3 3 5 5 5 F F F 24 480-499 5 5 5 7 7 7 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 6 6 5 8 8 7 F F T <--- 27 540-559 6 6 5 9 9 7 F F F <--- 28 560-579 5 5 5 7 7 7 F F F 29 580-599 5 5 5 7 7 7 F F F Max gen: 6, Max flush: 10 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 F F F 1 020-039 7 7 7 11 11 11 F F F 2 040-059 7 7 7 11 11 11 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 5 5 5 7 7 7 F F F 5 100-119 6 6 6 8 8 8 F F F 6 120-139 5 5 5 7 7 7 F F F 7 140-159 7 7 7 11 11 11 F F F 8 160-179 7 7 7 11 11 11 F F F 9 180-199 3 3 3 5 5 5 F F F 10 200-219 5 5 5 7 7 7 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 7 7 7 11 11 11 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 5 5 5 7 7 7 F F F 16 320-339 5 5 5 7 7 7 F F F 17 340-359 7 7 7 11 11 11 F F F 18 360-379 7 7 7 11 11 11 F F F 19 380-399 7 7 7 11 11 11 F F F 20 400-419 7 7 7 11 11 11 F F F 21 420-439 6 6 6 10 10 10 F F F 22 440-459 5 5 5 7 7 7 F F F 23 460-479 3 3 3 5 5 5 F F F 24 480-499 5 5 5 7 7 7 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 6 6 6 9 9 9 F F F 28 560-579 5 5 5 7 7 7 F F F 29 580-599 5 5 5 7 7 7 F F F Max gen: 7, Max flush: 11 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.968019433Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.976198567Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.976481198Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.976512216Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.976521135Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.976533074Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165} {"msg":"Crucible b1ba9372-9543-45d1-ac08-56d348055141 has session id: 60a95014-11bf-4962-90e4-450c195b8d7e","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.976939277Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"Upstairs opts: Upstairs UUID: b1ba9372-9543-45d1-ac08-56d348055141, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.976960665Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"Crucible stats registered with UUID: b1ba9372-9543-45d1-ac08-56d348055141","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.976971194Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:06.976984323Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977173071Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977190009Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977202649Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977319501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977337889Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165} {"msg":"b1ba9372-9543-45d1-ac08-56d348055141 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977361408Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977371087Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"client":"0","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977381196Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"client":"1","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977391216Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"client":"2","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977401445Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977414544Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977425143Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:59979)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.97748288Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"io task","client":"2","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55290)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977551295Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"io task","client":"0","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46580)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.977593862Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"io task","client":"1","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978237108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"client":"2","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978255627Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"client":"0","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978340021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"client":"1","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978572185Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978583105Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"[0]R flush_numbers[0..12]: [2, 11, 11, 2, 7, 8, 7, 11, 11, 5, 7, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978617232Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"downstairs","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"[0]R generation[0..12]: [2, 7, 7, 2, 5, 6, 5, 7, 7, 3, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978628612Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"downstairs","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978640161Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"downstairs","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"[1]R flush_numbers[0..12]: [2, 11, 11, 2, 7, 8, 7, 11, 11, 5, 7, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.97865086Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"downstairs","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"[1]R generation[0..12]: [2, 7, 7, 2, 5, 6, 5, 7, 7, 3, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978661589Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"downstairs","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978672969Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"downstairs","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"[2]R flush_numbers[0..12]: [2, 11, 11, 2, 7, 8, 7, 11, 11, 5, 7, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978683658Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"downstairs","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"[2]R generation[0..12]: [2, 7, 7, 2, 5, 6, 5, 7, 7, 3, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978694037Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"downstairs","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978705516Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"downstairs","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978715266Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"downstairs","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978725235Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"downstairs","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978734914Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"downstairs","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978746834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"":"downstairs","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978757033Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978766172Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"client":"0","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978776242Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"client":"1","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978786171Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"client":"2","":"downstairs","session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.97879646Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.97880594Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"b1ba9372-9543-45d1-ac08-56d348055141 is now active with session: 60a95014-11bf-4962-90e4-450c195b8d7e","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978816109Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978825608Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978833618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"session_id":"60a95014-11bf-4962-90e4-450c195b8d7e"} {"msg":"Activated sub_volume b1ba9372-9543-45d1-ac08-56d348055141","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978843067Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.978851526Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.979205162Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"task":"crutest"} 01/30 Write block 466 len 4096 data: 4 02/30 Read block 103 len 32768 03/30 Read block 133 len 4096 04/30 Flush 05/30 Write block 576 len 4096 data: 3 06/30 Write block 402 len 32768 data: 3 2 2 2 2 2 2 2 07/30 Write block 241 len 36864 data: 3 3 3 3 4 4 4 2 2 08/30 Read block 197 len 28672 09/30 Read block 304 len 40960 10/30 Read block 342 len 8192 11/30 Write block 95 len 24576 data: 3 3 3 3 3 2 12/30 Write block 463 len 16384 data: 4 4 4 5 13/30 Read block 41 len 28672 14/30 Read block 63 len 12288 15/30 Read block 364 len 28672 16/30 Flush 17/30 Write block 311 len 28672 data: 2 3 2 2 2 2 2 18/30 Flush 19/30 Write block 15 len 4096 data: 3 20/30 Read block 470 len 28672 21/30 Read block 193 len 4096 22/30 Read block 224 len 4096 23/30 Write block 496 len 8192 data: 2 2 24/30 Read block 501 len 32768 25/30 Write block 316 len 20480 data: 3 3 2 2 2 26/30 Read block 543 len 32768 27/30 Read block 79 len 40960 28/30 Read block 525 len 36864 29/30 Write block 455 len 16384 data: 3 3 5 5 30/30 Write block 355 len 20480 data: 2 2 3 3 3 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 1 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 26 2 30 Done 24 0 24 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1003)) Some(JobId(1017)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:06.987676156Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:06.987944508Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1165,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 2 2 2 T F T <--- 1 020-039 7 7 7 11 11 11 F F F 2 040-059 7 7 7 11 11 11 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 8 5 8 13 7 13 F F F <--- 5 100-119 8 6 8 13 8 13 F F F <--- 6 120-139 5 5 5 7 7 7 F F F 7 140-159 7 7 7 11 11 11 F F F 8 160-179 7 7 7 11 11 11 F F F 9 180-199 3 3 3 5 5 5 F F F 10 200-219 5 5 5 7 7 7 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 8 7 8 13 11 13 F F F <--- 13 260-279 2 2 2 4 4 4 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 8 5 8 14 7 14 T F T <--- 16 320-339 5 5 5 7 7 7 T F T <--- 17 340-359 7 7 7 11 11 11 T F T <--- 18 360-379 7 7 7 11 11 11 F F F 19 380-399 7 7 7 11 11 11 F F F 20 400-419 8 7 8 13 11 13 F F F <--- 21 420-439 6 6 6 10 10 10 F F F 22 440-459 5 5 5 7 7 7 T F T <--- 23 460-479 8 8 8 13 12 13 F F F <--- 24 480-499 5 5 5 7 7 7 T F T <--- 25 500-519 4 4 4 6 6 6 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 6 6 6 9 9 9 F F F 28 560-579 8 5 8 13 7 13 F F F <--- 29 580-599 5 5 5 7 7 7 F F F Max gen: 8, Max flush: 14 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 9 9 9 15 15 15 F F F 1 020-039 7 7 7 11 11 11 F F F 2 040-059 7 7 7 11 11 11 F F F 3 060-079 2 2 2 2 2 2 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 8 8 8 13 13 13 F F F 6 120-139 5 5 5 7 7 7 F F F 7 140-159 7 7 7 11 11 11 F F F 8 160-179 7 7 7 11 11 11 F F F 9 180-199 3 3 3 5 5 5 F F F 10 200-219 5 5 5 7 7 7 F F F 11 220-239 6 6 6 9 9 9 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 9 9 9 15 15 15 F F F 16 320-339 9 9 9 15 15 15 F F F 17 340-359 9 9 9 15 15 15 F F F 18 360-379 7 7 7 11 11 11 F F F 19 380-399 7 7 7 11 11 11 F F F 20 400-419 8 8 8 13 13 13 F F F 21 420-439 6 6 6 10 10 10 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 6 6 6 9 9 9 F F F 28 560-579 8 8 8 13 13 13 F F F 29 580-599 5 5 5 7 7 7 F F F Max gen: 9, Max flush: 15 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.226794441Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.235057019Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.235380237Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.235407275Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.235416404Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.235430393Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173} {"msg":"Crucible b614a154-f385-4498-80cf-af6f778dcf4f has session id: 993f280f-651c-4524-b12b-fe9cefd5a493","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.235756891Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"Upstairs opts: Upstairs UUID: b614a154-f385-4498-80cf-af6f778dcf4f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.235786119Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"Crucible stats registered with UUID: b614a154-f385-4498-80cf-af6f778dcf4f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.235797478Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:11.235807267Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.235981056Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.236013533Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.236025013Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.236111987Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.236130245Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173} {"msg":"b614a154-f385-4498-80cf-af6f778dcf4f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.236169403Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.236179222Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"client":"0","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.236189281Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"client":"1","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.236199231Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"client":"2","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.23620912Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.236222479Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.236276686Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"ds_connection connected from Ok(127.0.0.1:43965)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.236287895Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"io task","client":"1","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"ds_connection connected from Ok(127.0.0.1:62230)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.236298744Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"io task","client":"2","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"ds_connection connected from Ok(127.0.0.1:60790)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.236396787Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"io task","client":"0","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237021815Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"client":"1","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237040064Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"client":"0","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237051293Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"client":"2","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237348073Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237372971Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"[0]R flush_numbers[0..12]: [15, 11, 11, 2, 13, 13, 7, 11, 11, 5, 7, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.23738967Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"downstairs","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"[0]R generation[0..12]: [9, 7, 7, 2, 8, 8, 5, 7, 7, 3, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237401119Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"downstairs","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237412638Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"downstairs","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"[1]R flush_numbers[0..12]: [15, 11, 11, 2, 13, 13, 7, 11, 11, 5, 7, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237423697Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"downstairs","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"[1]R generation[0..12]: [9, 7, 7, 2, 8, 8, 5, 7, 7, 3, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237434327Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"downstairs","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237448816Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"downstairs","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"[2]R flush_numbers[0..12]: [15, 11, 11, 2, 13, 13, 7, 11, 11, 5, 7, 9]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237459785Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"downstairs","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"[2]R generation[0..12]: [9, 7, 7, 2, 8, 8, 5, 7, 7, 3, 5, 6]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237470434Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"downstairs","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237481704Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"downstairs","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237491453Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"downstairs","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237501552Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"downstairs","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237511372Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"downstairs","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237521231Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"":"downstairs","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.2375314Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.2375409Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"client":"0","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237551159Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"client":"1","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237561278Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"client":"2","":"downstairs","session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237571387Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237580967Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"b614a154-f385-4498-80cf-af6f778dcf4f is now active with session: 993f280f-651c-4524-b12b-fe9cefd5a493","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237591236Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237601795Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"session_id":"993f280f-651c-4524-b12b-fe9cefd5a493"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237611175Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173} {"msg":"Activated sub_volume b614a154-f385-4498-80cf-af6f778dcf4f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237619654Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237628204Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.237958281Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"task":"crutest"} 01/30 Write block 7 len 32768 data: 2 2 2 2 2 2 3 3 02/30 Read block 358 len 12288 03/30 Read block 534 len 32768 04/30 Write block 204 len 20480 data: 3 3 3 3 4 05/30 Read block 280 len 32768 06/30 Write block 368 len 8192 data: 2 2 07/30 Write block 91 len 12288 data: 2 2 3 08/30 Read block 397 len 16384 09/30 Write block 185 len 4096 data: 2 10/30 Write block 25 len 28672 data: 2 2 2 2 2 2 2 11/30 Write block 561 len 32768 data: 2 2 2 2 2 2 2 2 12/30 Read block 171 len 28672 13/30 Write block 86 len 12288 data: 2 2 2 14/30 Read block 350 len 28672 15/30 Write block 10 len 16384 data: 3 3 3 4 16/30 Read block 380 len 24576 17/30 Write block 185 len 32768 data: 3 2 2 3 3 3 3 3 18/30 Read block 245 len 36864 19/30 Read block 515 len 16384 20/30 Flush 21/30 Write block 298 len 40960 data: 2 2 2 2 2 2 2 2 2 2 22/30 Write block 61 len 20480 data: 2 2 2 2 2 23/30 Write block 230 len 28672 data: 2 2 2 2 2 3 3 24/30 Read block 45 len 32768 25/30 Write block 7 len 12288 data: 3 3 3 26/30 Write block 117 len 16384 data: 2 2 2 2 27/30 Write block 425 len 16384 data: 2 2 2 2 28/30 Read block 183 len 40960 29/30 Write block 280 len 20480 data: 2 2 2 2 2 30/30 Read block 326 len 20480 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Sent false 1001 Acked Read 3 Done Done Sent false 1002 Acked Read 8 Done Done Sent false 1003 Acked Write 5 Done Done Sent false 1004 Acked Read 8 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Write 5 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Write 5 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:11.246236668Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:11.246523338Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1173,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 10 10 9 16 16 15 T T F <--- 1 020-039 10 10 7 16 16 11 F F F <--- 2 040-059 7 7 7 11 11 11 F F F 3 060-079 2 2 2 2 2 2 T T F <--- 4 080-099 10 10 8 16 16 13 F F F <--- 5 100-119 8 8 8 13 13 13 T T F <--- 6 120-139 5 5 5 7 7 7 T T F <--- 7 140-159 7 7 7 11 11 11 F F F 8 160-179 7 7 7 11 11 11 F F F 9 180-199 10 10 3 16 16 5 F F F <--- 10 200-219 10 10 5 16 16 7 F F F <--- 11 220-239 6 6 6 9 9 9 T T F <--- 12 240-259 8 8 8 13 13 13 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 1 1 1 1 1 1 T T F <--- 15 300-319 9 9 9 15 15 15 T T F <--- 16 320-339 9 9 9 15 15 15 F F F 17 340-359 9 9 9 15 15 15 F F F 18 360-379 10 10 7 16 16 11 F F F <--- 19 380-399 7 7 7 11 11 11 F F F 20 400-419 8 8 8 13 13 13 F F F 21 420-439 6 6 6 10 10 10 T T F <--- 22 440-459 9 9 9 15 15 15 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 6 6 6 9 9 9 F F F 28 560-579 10 10 8 16 16 13 F F F <--- 29 580-599 5 5 5 7 7 7 F F F Max gen: 10, Max flush: 16 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 17 17 17 F F F 1 020-039 10 10 10 16 16 16 F F F 2 040-059 7 7 7 11 11 11 F F F 3 060-079 11 11 11 17 17 17 F F F 4 080-099 10 10 10 16 16 16 F F F 5 100-119 11 11 11 17 17 17 F F F 6 120-139 11 11 11 17 17 17 F F F 7 140-159 7 7 7 11 11 11 F F F 8 160-179 7 7 7 11 11 11 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 11 11 11 17 17 17 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 11 11 11 17 17 17 F F F 15 300-319 11 11 11 17 17 17 F F F 16 320-339 9 9 9 15 15 15 F F F 17 340-359 9 9 9 15 15 15 F F F 18 360-379 10 10 10 16 16 16 F F F 19 380-399 7 7 7 11 11 11 F F F 20 400-419 8 8 8 13 13 13 F F F 21 420-439 11 11 11 17 17 17 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 6 6 6 9 9 9 F F F 28 560-579 10 10 10 16 16 16 F F F 29 580-599 5 5 5 7 7 7 F F F Max gen: 11, Max flush: 17 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.493992665Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:15.501125749Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.501192195Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.508152301Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.508397675Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.508421443Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.508430642Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.508442532Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181} {"msg":"Crucible 98843fe7-5fa2-49c0-87c4-cf7c6354859d has session id: e6badd45-4900-459a-b9d6-1399f84cf9c0","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.508818846Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"Upstairs opts: Upstairs UUID: 98843fe7-5fa2-49c0-87c4-cf7c6354859d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.508848714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"Crucible stats registered with UUID: 98843fe7-5fa2-49c0-87c4-cf7c6354859d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.508862693Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:15.508872402Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.50905089Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509088468Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509100097Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509209949Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509228108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181} {"msg":"98843fe7-5fa2-49c0-87c4-cf7c6354859d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509267086Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509277175Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"client":"0","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509287224Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"client":"1","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509297344Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"client":"2","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509307423Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509318012Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509328341Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:37998)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509394507Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"io task","client":"0","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:33955)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509405846Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"io task","client":"1","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"ds_connection connected from Ok(127.0.0.1:59346)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.509418945Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"io task","client":"2","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510105459Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"client":"1","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510137756Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"client":"2","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510152525Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"client":"0","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510427287Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510465194Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"[0]R flush_numbers[0..12]: [17, 16, 11, 17, 16, 17, 17, 11, 11, 16, 16, 17]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.51051858Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"downstairs","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"[0]R generation[0..12]: [11, 10, 7, 11, 10, 11, 11, 7, 7, 10, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.51053017Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"downstairs","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510541569Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"downstairs","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"[1]R flush_numbers[0..12]: [17, 16, 11, 17, 16, 17, 17, 11, 11, 16, 16, 17]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510552348Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"downstairs","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"[1]R generation[0..12]: [11, 10, 7, 11, 10, 11, 11, 7, 7, 10, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510562937Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"downstairs","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510574257Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"downstairs","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"[2]R flush_numbers[0..12]: [17, 16, 11, 17, 16, 17, 17, 11, 11, 16, 16, 17]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510584936Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"downstairs","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"[2]R generation[0..12]: [11, 10, 7, 11, 10, 11, 11, 7, 7, 10, 10, 11]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510595505Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"downstairs","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510609404Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"downstairs","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510619314Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"downstairs","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510629323Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"downstairs","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510638942Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"downstairs","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510648742Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"":"downstairs","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510658711Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.51066781Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"client":"0","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.51067779Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"client":"1","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510687639Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"client":"2","":"downstairs","session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510697678Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510707148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"98843fe7-5fa2-49c0-87c4-cf7c6354859d is now active with session: e6badd45-4900-459a-b9d6-1399f84cf9c0","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510717537Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510727006Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510734946Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"session_id":"e6badd45-4900-459a-b9d6-1399f84cf9c0"} {"msg":"Activated sub_volume 98843fe7-5fa2-49c0-87c4-cf7c6354859d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510744295Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.510752664Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.511077182Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"task":"crutest"} 01/30 Write block 224 len 4096 data: 3 02/30 Flush 03/30 Write block 226 len 24576 data: 2 2 2 2 3 3 04/30 Flush 05/30 Write block 575 len 20480 data: 3 4 3 3 3 06/30 Write block 336 len 24576 data: 2 2 2 3 3 2 07/30 Read block 339 len 16384 08/30 Write block 151 len 4096 data: 3 09/30 Write block 39 len 36864 data: 3 4 4 4 3 3 3 3 3 10/30 Read block 181 len 12288 11/30 Read block 528 len 20480 12/30 Write block 44 len 8192 data: 4 4 13/30 Read block 548 len 4096 14/30 Flush 15/30 Write block 328 len 36864 data: 3 3 3 3 3 3 2 2 3 16/30 Read block 301 len 36864 17/30 Write block 284 len 24576 data: 3 2 2 2 2 2 18/30 Read block 325 len 40960 19/30 Write block 54 len 24576 data: 2 2 2 2 2 2 20/30 Read block 303 len 28672 21/30 Read block 28 len 32768 22/30 Write block 575 len 32768 data: 4 5 4 4 4 3 3 2 23/30 Read block 213 len 8192 24/30 Write block 568 len 32768 data: 3 2 2 2 2 3 3 5 25/30 Write block 231 len 28672 data: 4 3 3 3 4 4 2 26/30 Read block 356 len 12288 27/30 Read block 211 len 40960 28/30 Read block 515 len 36864 29/30 Read block 589 len 8192 30/30 Write block 482 len 36864 data: 2 2 2 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 6 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Write 6 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Write 6 Sent Done Done false 1019 Acked Read 7 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 0 27 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:15.520629492Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:15.520901784Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1181,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 17 17 17 F F F 1 020-039 10 12 12 16 20 20 F F F <--- 2 040-059 7 12 12 11 20 20 F T T <--- 3 060-079 11 11 11 17 17 17 F F F 4 080-099 10 10 10 16 16 16 F F F 5 100-119 11 11 11 17 17 17 F F F 6 120-139 11 11 11 17 17 17 F F F 7 140-159 7 12 12 11 20 20 F F F <--- 8 160-179 7 7 7 11 11 11 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 12 12 12 18 19 19 F T T <--- 12 240-259 8 8 8 13 13 13 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 11 11 11 17 17 17 F T T <--- 15 300-319 11 11 11 17 17 17 F F F 16 320-339 9 12 12 15 20 20 F T T <--- 17 340-359 9 12 12 15 20 20 F F F <--- 18 360-379 10 10 10 16 16 16 F F F 19 380-399 7 7 7 11 11 11 F F F 20 400-419 8 8 8 13 13 13 F F F 21 420-439 11 11 11 17 17 17 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 9 9 9 15 15 15 F T T <--- 25 500-519 4 4 4 6 6 6 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 6 6 6 9 9 9 F F F 28 560-579 10 12 12 16 20 20 F T T <--- 29 580-599 5 5 5 7 7 7 F T T <--- Max gen: 12, Max flush: 20 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 17 17 17 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 11 11 11 17 17 17 F F F 4 080-099 10 10 10 16 16 16 F F F 5 100-119 11 11 11 17 17 17 F F F 6 120-139 11 11 11 17 17 17 F F F 7 140-159 12 12 12 20 20 20 F F F 8 160-179 7 7 7 11 11 11 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 10 10 10 16 16 16 F F F 11 220-239 13 13 13 21 21 21 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 13 13 13 21 21 21 F F F 15 300-319 11 11 11 17 17 17 F F F 16 320-339 13 13 13 21 21 21 F F F 17 340-359 12 12 12 20 20 20 F F F 18 360-379 10 10 10 16 16 16 F F F 19 380-399 7 7 7 11 11 11 F F F 20 400-419 8 8 8 13 13 13 F F F 21 420-439 11 11 11 17 17 17 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 8 8 8 13 13 13 F F F 24 480-499 13 13 13 21 21 21 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 6 6 6 9 9 9 F F F 28 560-579 13 13 13 21 21 21 F F F 29 580-599 13 13 13 21 21 21 F F F Max gen: 13, Max flush: 21 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.762333821Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.770768057Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771046448Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771073007Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771082216Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771094185Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189} {"msg":"Crucible 41099b43-53ce-4322-ab22-4758b0a34e6d has session id: 59be718c-9db4-486a-bea2-deeb30449add","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771492158Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"Upstairs opts: Upstairs UUID: 41099b43-53ce-4322-ab22-4758b0a34e6d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771522506Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"Crucible stats registered with UUID: 41099b43-53ce-4322-ab22-4758b0a34e6d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771533835Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:19.771546504Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771711393Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771728322Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771738801Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771853623Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771879522Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189} {"msg":"41099b43-53ce-4322-ab22-4758b0a34e6d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771895301Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.77190478Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"client":"0","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771915109Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"client":"1","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771925069Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"client":"2","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771935088Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771954007Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771967636Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"ds_connection connected from Ok(127.0.0.1:33285)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.771978535Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"ds_connection connected from Ok(127.0.0.1:51910)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.772135984Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"ds_connection connected from Ok(127.0.0.1:63358)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.772168052Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.772697836Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"client":"0","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.772717365Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"client":"1","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.772808728Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"client":"2","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773107098Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773131127Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"[0]R flush_numbers[0..12]: [17, 20, 21, 17, 16, 17, 17, 20, 11, 16, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773142116Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"downstairs","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"[0]R generation[0..12]: [11, 12, 13, 11, 10, 11, 11, 12, 7, 10, 10, 13]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773155285Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"downstairs","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773166774Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"downstairs","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"[1]R flush_numbers[0..12]: [17, 20, 21, 17, 16, 17, 17, 20, 11, 16, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773177753Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"downstairs","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"[1]R generation[0..12]: [11, 12, 13, 11, 10, 11, 11, 12, 7, 10, 10, 13]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773188543Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"downstairs","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773199832Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"downstairs","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"[2]R flush_numbers[0..12]: [17, 20, 21, 17, 16, 17, 17, 20, 11, 16, 16, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773210711Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"downstairs","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"[2]R generation[0..12]: [11, 12, 13, 11, 10, 11, 11, 12, 7, 10, 10, 13]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.77322147Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"downstairs","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.77323326Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"downstairs","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773243219Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"downstairs","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773253318Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"downstairs","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773263098Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"downstairs","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773273077Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"":"downstairs","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773283426Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773292626Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"client":"0","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773304205Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"client":"1","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773314334Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"client":"2","":"downstairs","session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773324613Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773334203Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"41099b43-53ce-4322-ab22-4758b0a34e6d is now active with session: 59be718c-9db4-486a-bea2-deeb30449add","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773344632Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773353841Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"session_id":"59be718c-9db4-486a-bea2-deeb30449add"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773363421Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189} {"msg":"Activated sub_volume 41099b43-53ce-4322-ab22-4758b0a34e6d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.77337202Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.77338055Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.773707927Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"task":"crutest"} 01/30 Write block 286 len 12288 data: 3 3 3 02/30 Write block 553 len 32768 data: 3 3 3 3 2 2 2 2 03/30 Read block 531 len 32768 04/30 Read block 507 len 36864 05/30 Read block 383 len 20480 06/30 Write block 359 len 12288 data: 4 3 3 07/30 Flush 08/30 Write block 147 len 4096 data: 4 09/30 Read block 218 len 12288 10/30 Read block 433 len 32768 11/30 Read block 73 len 20480 12/30 Read block 353 len 40960 13/30 Write block 464 len 8192 data: 5 5 14/30 Read block 449 len 40960 15/30 Write block 476 len 8192 data: 2 2 16/30 Write block 210 len 36864 data: 4 4 3 3 3 4 4 3 3 17/30 Read block 142 len 28672 18/30 Read block 51 len 4096 19/30 Read block 353 len 32768 20/30 Read block 301 len 4096 21/30 Read block 417 len 36864 22/30 Read block 113 len 36864 23/30 Write block 344 len 40960 data: 2 2 2 2 2 2 2 2 2 2 24/30 Write block 64 len 4096 data: 3 25/30 Flush 26/30 Write block 97 len 36864 data: 4 4 4 3 2 2 2 2 2 27/30 Read block 322 len 40960 28/30 Read block 557 len 12288 29/30 Read block 271 len 28672 30/30 Read block 137 len 32768 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 9 Done Done Sent false 1004 Acked Read 5 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 1 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 27 28 Done 30 29 3 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:19.782582673Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:19.782866484Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1189,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 17 17 17 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 14 14 11 23 23 17 F F F <--- 4 080-099 10 10 10 16 16 16 T T F <--- 5 100-119 11 11 11 17 17 17 T T F <--- 6 120-139 11 11 11 17 17 17 F F F 7 140-159 14 14 12 23 23 20 F F F <--- 8 160-179 7 7 7 11 11 11 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 14 14 10 23 23 16 F F F <--- 11 220-239 13 13 13 21 21 21 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 14 14 13 22 22 21 F F T <--- 15 300-319 11 11 11 17 17 17 F F F 16 320-339 13 13 13 21 21 21 F F F 17 340-359 14 14 12 23 23 20 F F F <--- 18 360-379 14 14 10 22 22 16 F F F <--- 19 380-399 7 7 7 11 11 11 F F F 20 400-419 8 8 8 13 13 13 F F F 21 420-439 11 11 11 17 17 17 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 14 14 8 23 23 13 F F F <--- 24 480-499 13 13 13 21 21 21 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 14 14 6 22 22 9 F F T <--- 28 560-579 14 14 13 22 22 21 F F T <--- 29 580-599 13 13 13 21 21 21 F F F Max gen: 14, Max flush: 23 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 17 17 17 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 13 13 13 21 21 21 F F F 3 060-079 14 14 14 23 23 23 F F F 4 080-099 15 15 15 24 24 24 F F F 5 100-119 15 15 15 24 24 24 F F F 6 120-139 11 11 11 17 17 17 F F F 7 140-159 14 14 14 23 23 23 F F F 8 160-179 7 7 7 11 11 11 F F F 9 180-199 10 10 10 16 16 16 F F F 10 200-219 14 14 14 23 23 23 F F F 11 220-239 13 13 13 21 21 21 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 14 14 14 22 22 22 F F F 15 300-319 11 11 11 17 17 17 F F F 16 320-339 13 13 13 21 21 21 F F F 17 340-359 14 14 14 23 23 23 F F F 18 360-379 14 14 14 22 22 22 F F F 19 380-399 7 7 7 11 11 11 F F F 20 400-419 8 8 8 13 13 13 F F F 21 420-439 11 11 11 17 17 17 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 14 14 14 23 23 23 F F F 24 480-499 13 13 13 21 21 21 F F F 25 500-519 4 4 4 6 6 6 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 14 14 14 22 22 22 F F F 28 560-579 14 14 14 22 22 22 F F F 29 580-599 13 13 13 21 21 21 F F F Max gen: 15, Max flush: 24 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.025240077Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.033393562Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.033677493Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.033703121Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.033712271Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.03372414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197} {"msg":"Crucible bf084401-fd9b-4d9e-a6c6-4e079eb5788a has session id: 1dc21782-f5bc-473f-8368-3889c1bd797f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034081266Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"Upstairs opts: Upstairs UUID: bf084401-fd9b-4d9e-a6c6-4e079eb5788a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034100384Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"Crucible stats registered with UUID: bf084401-fd9b-4d9e-a6c6-4e079eb5788a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034110914Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:24.034124833Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034295431Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034326469Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034337978Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034432232Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.03445812Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197} {"msg":"bf084401-fd9b-4d9e-a6c6-4e079eb5788a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034474079Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034483358Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"client":"0","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034493578Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"client":"1","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034503827Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"client":"2","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034513826Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034526765Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034539594Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54903)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034579482Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"io task","client":"2","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"ds_connection connected from Ok(127.0.0.1:47278)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034591001Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"io task","client":"1","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"ds_connection connected from Ok(127.0.0.1:59339)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.034734761Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"io task","client":"0","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035295203Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"client":"0","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035309112Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"client":"1","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035380017Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"client":"2","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035623481Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035659658Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"[0]R flush_numbers[0..12]: [17, 20, 21, 23, 24, 24, 17, 23, 11, 16, 23, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035720684Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"downstairs","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"[0]R generation[0..12]: [11, 12, 13, 14, 15, 15, 11, 14, 7, 10, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035733543Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"downstairs","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035745222Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"downstairs","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"[1]R flush_numbers[0..12]: [17, 20, 21, 23, 24, 24, 17, 23, 11, 16, 23, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035756222Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"downstairs","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"[1]R generation[0..12]: [11, 12, 13, 14, 15, 15, 11, 14, 7, 10, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035766881Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"downstairs","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.03577828Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"downstairs","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"[2]R flush_numbers[0..12]: [17, 20, 21, 23, 24, 24, 17, 23, 11, 16, 23, 21]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035792049Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"downstairs","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"[2]R generation[0..12]: [11, 12, 13, 14, 15, 15, 11, 14, 7, 10, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035802789Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"downstairs","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035814108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"downstairs","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035823797Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"downstairs","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035833836Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"downstairs","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"Next flush: 25","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035843556Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"downstairs","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035853235Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"":"downstairs","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035863634Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035872654Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"client":"0","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035882883Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"client":"1","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035893082Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"client":"2","":"downstairs","session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035908651Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035918221Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"bf084401-fd9b-4d9e-a6c6-4e079eb5788a is now active with session: 1dc21782-f5bc-473f-8368-3889c1bd797f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.03592844Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035937649Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"session_id":"1dc21782-f5bc-473f-8368-3889c1bd797f"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035947069Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197} {"msg":"Activated sub_volume bf084401-fd9b-4d9e-a6c6-4e079eb5788a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035956608Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.035965038Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.036301415Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"task":"crutest"} 01/30 Write block 221 len 40960 data: 4 4 3 4 2 3 3 3 3 4 02/30 Write block 208 len 20480 data: 5 4 5 5 4 03/30 Write block 569 len 28672 data: 3 3 3 3 4 4 6 04/30 Read block 149 len 20480 05/30 Write block 431 len 20480 data: 4 5 5 4 4 06/30 Read block 24 len 40960 07/30 Read block 334 len 28672 08/30 Write block 184 len 32768 data: 2 4 3 3 4 4 4 4 09/30 Read block 429 len 12288 10/30 Write block 509 len 32768 data: 2 2 2 2 2 2 2 2 11/30 Read block 473 len 24576 12/30 Read block 574 len 32768 13/30 Write block 300 len 36864 data: 3 3 3 3 3 3 3 3 2 14/30 Write block 158 len 36864 data: 2 4 3 3 3 3 3 3 3 15/30 Write block 50 len 8192 data: 2 2 16/30 Flush 17/30 Write block 337 len 8192 data: 3 3 18/30 Read block 247 len 40960 19/30 Read block 85 len 20480 20/30 Write block 166 len 40960 data: 4 4 3 3 3 3 2 2 2 2 21/30 Write block 17 len 28672 data: 3 3 3 3 3 2 2 22/30 Read block 425 len 8192 23/30 Read block 391 len 20480 24/30 Write block 424 len 32768 data: 2 3 3 3 3 3 3 5 25/30 Write block 435 len 36864 data: 5 4 3 3 2 2 2 2 2 26/30 Read block 558 len 36864 27/30 Read block 245 len 8192 28/30 Read block 0 len 40960 29/30 Read block 343 len 16384 30/30 Write block 415 len 36864 data: 2 2 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 8 Done Done Done false 1008 Acked Read 3 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Read 6 Done Done Sent false 1011 Acked Read 8 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 2 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 20 22 Done 29 29 10 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:24.045787159Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:24.04607268Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1197,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 11 11 11 17 17 17 T T F <--- 1 020-039 12 12 12 20 20 20 T T F <--- 2 040-059 16 16 13 25 25 21 F F F <--- 3 060-079 14 14 14 23 23 23 F F F 4 080-099 15 15 15 24 24 24 F F F 5 100-119 15 15 15 24 24 24 F F F 6 120-139 11 11 11 17 17 17 F F F 7 140-159 16 16 14 25 25 23 F F F <--- 8 160-179 16 16 7 25 25 11 T T F <--- 9 180-199 16 16 10 25 25 16 F F T <--- 10 200-219 16 16 14 25 25 23 F F T <--- 11 220-239 16 16 13 25 25 21 F F T <--- 12 240-259 8 8 8 13 13 13 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 14 14 14 22 22 22 F F F 15 300-319 16 16 11 25 25 17 F F F <--- 16 320-339 13 13 13 21 21 21 T T F <--- 17 340-359 14 14 14 23 23 23 F F F 18 360-379 14 14 14 22 22 22 F F F 19 380-399 7 7 7 11 11 11 F F F 20 400-419 8 8 8 13 13 13 T T F <--- 21 420-439 16 16 11 25 25 17 T T T <--- 22 440-459 9 9 9 15 15 15 T T F <--- 23 460-479 14 14 14 23 23 23 F F F 24 480-499 13 13 13 21 21 21 F F F 25 500-519 16 16 4 25 25 6 F F T <--- 26 520-539 6 6 6 8 8 8 F F F 27 540-559 14 14 14 22 22 22 F F F 28 560-579 16 16 14 25 25 22 F F T <--- 29 580-599 13 13 13 21 21 21 F F F Max gen: 16, Max flush: 25 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 26 26 26 F F F 1 020-039 17 17 17 26 26 26 F F F 2 040-059 16 16 16 25 25 25 F F F 3 060-079 14 14 14 23 23 23 F F F 4 080-099 15 15 15 24 24 24 F F F 5 100-119 15 15 15 24 24 24 F F F 6 120-139 11 11 11 17 17 17 F F F 7 140-159 16 16 16 25 25 25 F F F 8 160-179 17 17 17 26 26 26 F F F 9 180-199 16 16 16 25 25 25 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 16 16 16 25 25 25 F F F 12 240-259 8 8 8 13 13 13 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 14 14 14 22 22 22 F F F 15 300-319 16 16 16 25 25 25 F F F 16 320-339 17 17 17 26 26 26 F F F 17 340-359 14 14 14 23 23 23 F F F 18 360-379 14 14 14 22 22 22 F F F 19 380-399 7 7 7 11 11 11 F F F 20 400-419 17 17 17 26 26 26 F F F 21 420-439 17 17 17 26 26 26 F F F 22 440-459 17 17 17 26 26 26 F F F 23 460-479 14 14 14 23 23 23 F F F 24 480-499 13 13 13 21 21 21 F F F 25 500-519 16 16 16 25 25 25 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 14 14 14 22 22 22 F F F 28 560-579 16 16 16 25 25 25 F F F 29 580-599 13 13 13 21 21 21 F F F Max gen: 17, Max flush: 26 Begin loop 9 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.297936177Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.306206015Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.306507354Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.306533422Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.306542782Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.306554591Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205} {"msg":"Crucible cb899a76-0d72-4464-9a04-1ea21fe7636d has session id: 119011d0-1b9d-447d-a368-1ae5e6821d67","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.306901927Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"Upstairs opts: Upstairs UUID: cb899a76-0d72-4464-9a04-1ea21fe7636d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.306920706Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"Crucible stats registered with UUID: cb899a76-0d72-4464-9a04-1ea21fe7636d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.306931265Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:28.306944734Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.307131072Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.30716171Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.307175449Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.307273362Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.3072985Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205} {"msg":"cb899a76-0d72-4464-9a04-1ea21fe7636d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.307326438Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.307336328Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"client":"0","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.307346407Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"client":"1","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.307356446Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"client":"2","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.307366586Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.307381555Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.307435291Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"ds_connection connected from Ok(127.0.0.1:53367)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.30745293Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"io task","client":"2","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"ds_connection connected from Ok(127.0.0.1:46106)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.307474888Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"io task","client":"1","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"ds_connection connected from Ok(127.0.0.1:44992)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.307485738Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"io task","client":"0","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308099976Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"client":"1","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308131294Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"client":"0","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308203039Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"client":"2","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308415414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308438243Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"[0]R flush_numbers[0..12]: [26, 26, 25, 23, 24, 24, 17, 25, 26, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308496809Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"downstairs","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"[0]R generation[0..12]: [17, 17, 16, 14, 15, 15, 11, 16, 17, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308508578Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"downstairs","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308520107Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"downstairs","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"[1]R flush_numbers[0..12]: [26, 26, 25, 23, 24, 24, 17, 25, 26, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308530886Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"downstairs","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"[1]R generation[0..12]: [17, 17, 16, 14, 15, 15, 11, 16, 17, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308541686Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"downstairs","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308552995Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"downstairs","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"[2]R flush_numbers[0..12]: [26, 26, 25, 23, 24, 24, 17, 25, 26, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308563704Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"downstairs","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"[2]R generation[0..12]: [17, 17, 16, 14, 15, 15, 11, 16, 17, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308574383Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"downstairs","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308585563Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"downstairs","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308595132Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"downstairs","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308605161Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"downstairs","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308614961Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"downstairs","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.30862745Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"":"downstairs","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308637909Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308646929Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"client":"0","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308656778Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"client":"1","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308666767Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"client":"2","":"downstairs","session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308676807Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308686206Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"cb899a76-0d72-4464-9a04-1ea21fe7636d is now active with session: 119011d0-1b9d-447d-a368-1ae5e6821d67","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308696465Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308705945Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308713994Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"session_id":"119011d0-1b9d-447d-a368-1ae5e6821d67"} {"msg":"Activated sub_volume cb899a76-0d72-4464-9a04-1ea21fe7636d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308723583Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.308732053Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.309094378Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"task":"crutest"} 01/30 Write block 454 len 20480 data: 3 4 4 6 6 02/30 Write block 54 len 16384 data: 3 3 3 3 03/30 Write block 434 len 28672 data: 5 6 5 4 4 3 3 04/30 Read block 117 len 4096 05/30 Read block 80 len 24576 06/30 Read block 572 len 40960 07/30 Write block 242 len 40960 data: 4 4 4 5 5 5 3 3 2 2 08/30 Write block 19 len 16384 data: 4 4 4 3 09/30 Read block 528 len 32768 10/30 Flush 11/30 Write block 298 len 4096 data: 3 12/30 Read block 248 len 20480 13/30 Read block 232 len 8192 14/30 Write block 106 len 36864 data: 2 2 4 4 4 4 3 3 3 15/30 Write block 377 len 28672 data: 3 4 4 3 3 3 3 16/30 Read block 573 len 32768 17/30 Write block 353 len 36864 data: 3 2 3 3 4 4 5 4 4 18/30 Write block 361 len 20480 data: 5 3 3 3 3 19/30 Read block 315 len 32768 20/30 Read block 565 len 20480 21/30 Read block 294 len 32768 22/30 Read block 498 len 8192 23/30 Read block 176 len 24576 24/30 Read block 304 len 12288 25/30 Write block 14 len 32768 data: 4 4 3 4 4 5 5 5 26/30 Write block 56 len 28672 data: 4 4 3 3 2 3 3 27/30 Write block 358 len 36864 data: 5 6 5 6 4 4 4 4 3 28/30 Write block 585 len 12288 data: 2 2 2 29/30 Read block 559 len 16384 30/30 Read block 138 len 28672 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 4 Done Done Sent false 1002 Acked Write 7 Done Done Sent false 1003 Acked Read 1 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Write 10 Done Done Sent false 1007 Acked Write 4 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Write 7 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 29 29 Done 30 30 1 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:28.31803249Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:28.318303772Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1205,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 17 27 27 26 T T F <--- 1 020-039 18 18 17 27 27 26 T T F <--- 2 040-059 18 18 16 27 27 25 T T F <--- 3 060-079 14 14 14 23 23 23 T T F <--- 4 080-099 15 15 15 24 24 24 F F F 5 100-119 15 15 15 24 24 24 T T F <--- 6 120-139 11 11 11 17 17 17 F F F 7 140-159 16 16 16 25 25 25 F F F 8 160-179 17 17 17 26 26 26 F F F 9 180-199 16 16 16 25 25 25 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 16 16 16 25 25 25 F F F 12 240-259 18 18 8 27 27 13 F F F <--- 13 260-279 2 2 2 4 4 4 F F F 14 280-299 14 14 14 22 22 22 T T F <--- 15 300-319 16 16 16 25 25 25 F F F 16 320-339 17 17 17 26 26 26 F F F 17 340-359 14 14 14 23 23 23 T T F <--- 18 360-379 14 14 14 22 22 22 T T F <--- 19 380-399 7 7 7 11 11 11 T T F <--- 20 400-419 17 17 17 26 26 26 F F F 21 420-439 18 18 17 27 27 26 F F F <--- 22 440-459 18 18 17 27 27 26 F F T <--- 23 460-479 14 14 14 23 23 23 F F F 24 480-499 13 13 13 21 21 21 F F F 25 500-519 16 16 16 25 25 25 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 14 14 14 22 22 22 F F F 28 560-579 16 16 16 25 25 25 F F F 29 580-599 13 13 13 21 21 21 T T F <--- Max gen: 18, Max flush: 27 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 19 19 19 28 28 28 F F F 1 020-039 19 19 19 28 28 28 F F F 2 040-059 19 19 19 28 28 28 F F F 3 060-079 19 19 19 28 28 28 F F F 4 080-099 15 15 15 24 24 24 F F F 5 100-119 19 19 19 28 28 28 F F F 6 120-139 11 11 11 17 17 17 F F F 7 140-159 16 16 16 25 25 25 F F F 8 160-179 17 17 17 26 26 26 F F F 9 180-199 16 16 16 25 25 25 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 16 16 16 25 25 25 F F F 12 240-259 18 18 18 27 27 27 F F F 13 260-279 2 2 2 4 4 4 F F F 14 280-299 19 19 19 28 28 28 F F F 15 300-319 16 16 16 25 25 25 F F F 16 320-339 17 17 17 26 26 26 F F F 17 340-359 19 19 19 28 28 28 F F F 18 360-379 19 19 19 28 28 28 F F F 19 380-399 19 19 19 28 28 28 F F F 20 400-419 17 17 17 26 26 26 F F F 21 420-439 18 18 18 27 27 27 F F F 22 440-459 18 18 18 27 27 27 F F F 23 460-479 14 14 14 23 23 23 F F F 24 480-499 13 13 13 21 21 21 F F F 25 500-519 16 16 16 25 25 25 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 14 14 14 22 22 22 F F F 28 560-579 16 16 16 25 25 25 F F F 29 580-599 19 19 19 28 28 28 F F F Max gen: 19, Max flush: 28 Begin loop 10 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.562433425Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.570544493Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.570845173Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.57088186Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.57089427Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.570906619Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213} {"msg":"Crucible 9e487b3e-36c7-4c60-916d-a80a3d311ee1 has session id: 25c282e6-9e9c-4602-90a2-e6cc2ad952d3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571246016Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"Upstairs opts: Upstairs UUID: 9e487b3e-36c7-4c60-916d-a80a3d311ee1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571289543Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"Crucible stats registered with UUID: 9e487b3e-36c7-4c60-916d-a80a3d311ee1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571301192Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:32.571314231Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.57148218Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571507008Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571518087Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.57162892Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571646258Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213} {"msg":"9e487b3e-36c7-4c60-916d-a80a3d311ee1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571661257Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571670757Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"client":"0","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571681036Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"client":"1","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571693255Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"client":"2","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571703364Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571715084Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571726883Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:58661)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.5717704Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"io task","client":"2","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:57250)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571782379Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"io task","client":"1","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:47904)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.571793188Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"io task","client":"0","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572438834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"client":"0","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572457613Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"client":"1","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572468962Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"client":"2","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572754443Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572765782Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"[0]R flush_numbers[0..12]: [28, 28, 28, 28, 24, 28, 17, 25, 26, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.57280391Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"downstairs","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"[0]R generation[0..12]: [19, 19, 19, 19, 15, 19, 11, 16, 17, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572815279Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"downstairs","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572826538Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"downstairs","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"[1]R flush_numbers[0..12]: [28, 28, 28, 28, 24, 28, 17, 25, 26, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572838507Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"downstairs","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"[1]R generation[0..12]: [19, 19, 19, 19, 15, 19, 11, 16, 17, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572849087Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"downstairs","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572860266Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"downstairs","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"[2]R flush_numbers[0..12]: [28, 28, 28, 28, 24, 28, 17, 25, 26, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572871055Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"downstairs","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"[2]R generation[0..12]: [19, 19, 19, 19, 15, 19, 11, 16, 17, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572881694Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"downstairs","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572892894Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"downstairs","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572902643Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"downstairs","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572912582Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"downstairs","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572922262Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"downstairs","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572932171Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"":"downstairs","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.57294226Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.57295136Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"client":"0","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572961269Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"client":"1","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572971128Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"client":"2","":"downstairs","session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572981368Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.572992177Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"9e487b3e-36c7-4c60-916d-a80a3d311ee1 is now active with session: 25c282e6-9e9c-4602-90a2-e6cc2ad952d3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.573002606Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.573012085Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.573020085Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"session_id":"25c282e6-9e9c-4602-90a2-e6cc2ad952d3"} {"msg":"Activated sub_volume 9e487b3e-36c7-4c60-916d-a80a3d311ee1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.573029554Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.573038004Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.573349622Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"task":"crutest"} 01/30 Write block 143 len 8192 data: 3 3 02/30 Read block 39 len 8192 03/30 Flush 04/30 Write block 261 len 24576 data: 2 2 2 2 2 2 05/30 Write block 321 len 8192 data: 2 2 06/30 Write block 269 len 4096 data: 2 07/30 Write block 519 len 4096 data: 2 08/30 Read block 42 len 20480 09/30 Write block 104 len 4096 data: 3 10/30 Read block 507 len 4096 11/30 Write block 4 len 12288 data: 2 2 2 12/30 Write block 55 len 32768 data: 4 5 5 4 4 3 4 4 13/30 Flush 14/30 Write block 454 len 28672 data: 4 5 5 7 7 5 4 15/30 Read block 446 len 20480 16/30 Read block 488 len 24576 17/30 Write block 547 len 36864 data: 2 2 4 3 3 3 4 4 4 18/30 Flush 19/30 Write block 150 len 12288 data: 4 4 3 20/30 Read block 577 len 36864 21/30 Read block 341 len 28672 22/30 Write block 319 len 24576 data: 3 3 3 3 2 2 23/30 Write block 106 len 12288 data: 3 3 5 24/30 Read block 48 len 4096 25/30 Read block 560 len 36864 26/30 Write block 378 len 4096 data: 5 27/30 Read block 262 len 28672 28/30 Read block 414 len 4096 29/30 Read block 576 len 40960 30/30 Read block 342 len 32768 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Read 2 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Write 2 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Write 1 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Write 1 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Write 3 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:32.581141512Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:32.581434172Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1213,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 19 30 30 28 F F F <--- 1 020-039 19 19 19 28 28 28 F F F 2 040-059 20 20 19 30 30 28 F F F <--- 3 060-079 20 20 19 30 30 28 F F F <--- 4 080-099 15 15 15 24 24 24 F F F 5 100-119 20 20 19 30 30 28 T T F <--- 6 120-139 11 11 11 17 17 17 F F F 7 140-159 20 20 16 29 29 25 T T F <--- 8 160-179 17 17 17 26 26 26 F F F 9 180-199 16 16 16 25 25 25 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 16 16 16 25 25 25 F F F 12 240-259 18 18 18 27 27 27 F F F 13 260-279 20 20 2 30 30 4 F F F <--- 14 280-299 19 19 19 28 28 28 F F F 15 300-319 16 16 16 25 25 25 T T F <--- 16 320-339 20 20 17 30 30 26 T T F <--- 17 340-359 19 19 19 28 28 28 F F F 18 360-379 19 19 19 28 28 28 T T F <--- 19 380-399 19 19 19 28 28 28 F F F 20 400-419 17 17 17 26 26 26 F F F 21 420-439 18 18 18 27 27 27 F F F 22 440-459 20 20 18 31 31 27 F F F <--- 23 460-479 20 20 14 31 31 23 F F F <--- 24 480-499 13 13 13 21 21 21 F F F 25 500-519 20 20 16 30 30 25 F F F <--- 26 520-539 6 6 6 8 8 8 F F F 27 540-559 20 20 14 31 31 22 F F F <--- 28 560-579 16 16 16 25 25 25 F F F 29 580-599 19 19 19 28 28 28 F F F Max gen: 20, Max flush: 31 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 19 19 19 28 28 28 F F F 2 040-059 20 20 20 30 30 30 F F F 3 060-079 20 20 20 30 30 30 F F F 4 080-099 15 15 15 24 24 24 F F F 5 100-119 21 21 21 32 32 32 F F F 6 120-139 11 11 11 17 17 17 F F F 7 140-159 21 21 21 32 32 32 F F F 8 160-179 17 17 17 26 26 26 F F F 9 180-199 16 16 16 25 25 25 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 16 16 16 25 25 25 F F F 12 240-259 18 18 18 27 27 27 F F F 13 260-279 20 20 20 30 30 30 F F F 14 280-299 19 19 19 28 28 28 F F F 15 300-319 21 21 21 32 32 32 F F F 16 320-339 21 21 21 32 32 32 F F F 17 340-359 19 19 19 28 28 28 F F F 18 360-379 21 21 21 32 32 32 F F F 19 380-399 19 19 19 28 28 28 F F F 20 400-419 17 17 17 26 26 26 F F F 21 420-439 18 18 18 27 27 27 F F F 22 440-459 20 20 20 31 31 31 F F F 23 460-479 20 20 20 31 31 31 F F F 24 480-499 13 13 13 21 21 21 F F F 25 500-519 20 20 20 30 30 30 F F F 26 520-539 6 6 6 8 8 8 F F F 27 540-559 20 20 20 31 31 31 F F F 28 560-579 16 16 16 25 25 25 F F F 29 580-599 19 19 19 28 28 28 F F F Max gen: 21, Max flush: 32 Begin loop 11 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.826703098Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.834873693Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.835165433Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.835196911Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.83520601Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.835218329Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221} {"msg":"Crucible 4a808288-bbc8-44ef-a03b-5ecc2de59853 has session id: 70030ce0-41c7-4d65-aad8-39ec3de1e1be","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.835629221Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"Upstairs opts: Upstairs UUID: 4a808288-bbc8-44ef-a03b-5ecc2de59853, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.83565164Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"Crucible stats registered with UUID: 4a808288-bbc8-44ef-a03b-5ecc2de59853","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.835662239Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:36.835676408Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.835849196Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.835878594Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.835889683Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.836030054Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.836042983Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221} {"msg":"4a808288-bbc8-44ef-a03b-5ecc2de59853 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.836054152Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.836063422Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"client":"0","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.836073461Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"client":"1","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.83608345Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"client":"2","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.83609352Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.836114158Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.836124847Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"ds_connection connected from Ok(127.0.0.1:41096)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.836199922Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"io task","client":"1","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"ds_connection connected from Ok(127.0.0.1:38628)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.836211522Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"io task","client":"0","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"ds_connection connected from Ok(127.0.0.1:63860)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.836305925Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"io task","client":"2","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.836859477Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"client":"1","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.836877296Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"client":"0","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837003758Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"client":"2","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837233612Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837243621Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"[0]R flush_numbers[0..12]: [30, 28, 30, 30, 24, 32, 17, 32, 26, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837289308Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"downstairs","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"[0]R generation[0..12]: [20, 19, 20, 20, 15, 21, 11, 21, 17, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837300717Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"downstairs","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837312107Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"downstairs","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"[1]R flush_numbers[0..12]: [30, 28, 30, 30, 24, 32, 17, 32, 26, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837322796Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"downstairs","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"[1]R generation[0..12]: [20, 19, 20, 20, 15, 21, 11, 21, 17, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837333435Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"downstairs","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837344604Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"downstairs","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"[2]R flush_numbers[0..12]: [30, 28, 30, 30, 24, 32, 17, 32, 26, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837355344Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"downstairs","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"[2]R generation[0..12]: [20, 19, 20, 20, 15, 21, 11, 21, 17, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837365853Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"downstairs","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837377142Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"downstairs","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837388701Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"downstairs","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837398731Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"downstairs","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.83740846Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"downstairs","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837418459Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"":"downstairs","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837428889Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837438078Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"client":"0","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837448597Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"client":"1","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837458907Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"client":"2","":"downstairs","session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837469276Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837478905Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"4a808288-bbc8-44ef-a03b-5ecc2de59853 is now active with session: 70030ce0-41c7-4d65-aad8-39ec3de1e1be","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837489285Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837498654Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837506573Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"session_id":"70030ce0-41c7-4d65-aad8-39ec3de1e1be"} {"msg":"Activated sub_volume 4a808288-bbc8-44ef-a03b-5ecc2de59853","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837516133Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.837524482Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.83784522Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"task":"crutest"} 01/30 Flush 02/30 Write block 137 len 24576 data: 2 2 2 2 3 3 03/30 Write block 60 len 40960 data: 4 5 5 3 4 3 2 2 2 2 04/30 Write block 167 len 16384 data: 5 4 4 4 05/30 Write block 525 len 12288 data: 2 3 3 06/30 Read block 488 len 32768 07/30 Write block 165 len 16384 data: 4 5 6 5 08/30 Write block 33 len 28672 data: 2 2 2 2 2 3 4 09/30 Write block 114 len 28672 data: 4 3 3 3 3 3 3 10/30 Read block 1 len 28672 11/30 Read block 255 len 20480 12/30 Write block 111 len 4096 data: 5 13/30 Write block 591 len 8192 data: 2 2 14/30 Read block 460 len 12288 15/30 Read block 9 len 36864 16/30 Write block 45 len 16384 data: 5 4 4 3 17/30 Write block 313 len 24576 data: 3 3 3 4 4 3 18/30 Write block 559 len 4096 data: 3 19/30 Read block 165 len 20480 20/30 Read block 447 len 20480 21/30 Read block 91 len 8192 22/30 Flush 23/30 Write block 74 len 4096 data: 2 24/30 Read block 283 len 20480 25/30 Read block 589 len 28672 26/30 Write block 349 len 24576 data: 3 3 3 3 4 3 27/30 Read block 545 len 4096 28/30 Write block 170 len 28672 data: 5 4 3 3 3 3 2 29/30 Read block 299 len 24576 30/30 Read block 516 len 28672 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 6 Done Done Done false 1002 Acked Write 10 Done Done Sent false 1003 Acked Write 4 Done Done Sent false 1004 Acked Write 3 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Write 4 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Read 3 Done Done Sent false 1014 Acked Read 9 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 28 28 Done 29 29 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:36.84578485Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:36.84608807Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1221,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 22 22 19 34 34 28 F F F <--- 2 040-059 22 22 20 34 34 30 F F F <--- 3 060-079 22 22 20 34 34 30 T T F <--- 4 080-099 15 15 15 24 24 24 F F F 5 100-119 22 22 21 34 34 32 F F F <--- 6 120-139 22 22 11 34 34 17 F F T <--- 7 140-159 22 22 21 34 34 32 F F T <--- 8 160-179 22 22 17 34 34 26 T T F <--- 9 180-199 16 16 16 25 25 25 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 16 16 16 25 25 25 F F F 12 240-259 18 18 18 27 27 27 F F F 13 260-279 20 20 20 30 30 30 F F F 14 280-299 19 19 19 28 28 28 F F F 15 300-319 22 22 21 34 34 32 F F F <--- 16 320-339 21 21 21 32 32 32 F F F 17 340-359 19 19 19 28 28 28 T T F <--- 18 360-379 21 21 21 32 32 32 F F F 19 380-399 19 19 19 28 28 28 F F F 20 400-419 17 17 17 26 26 26 F F F 21 420-439 18 18 18 27 27 27 F F F 22 440-459 20 20 20 31 31 31 F F F 23 460-479 20 20 20 31 31 31 F F F 24 480-499 13 13 13 21 21 21 F F F 25 500-519 20 20 20 30 30 30 F F F 26 520-539 22 22 6 34 34 8 F F F <--- 27 540-559 22 22 20 34 34 31 F F F <--- 28 560-579 16 16 16 25 25 25 F F F 29 580-599 22 22 19 34 34 28 F F F <--- Max gen: 22, Max flush: 34 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 22 22 22 34 34 34 F F F 2 040-059 22 22 22 34 34 34 F F F 3 060-079 23 23 23 35 35 35 F F F 4 080-099 15 15 15 24 24 24 F F F 5 100-119 22 22 22 34 34 34 F F F 6 120-139 22 22 22 34 34 34 F F F 7 140-159 22 22 22 34 34 34 F F F 8 160-179 23 23 23 35 35 35 F F F 9 180-199 16 16 16 25 25 25 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 16 16 16 25 25 25 F F F 12 240-259 18 18 18 27 27 27 F F F 13 260-279 20 20 20 30 30 30 F F F 14 280-299 19 19 19 28 28 28 F F F 15 300-319 22 22 22 34 34 34 F F F 16 320-339 21 21 21 32 32 32 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 21 21 21 32 32 32 F F F 19 380-399 19 19 19 28 28 28 F F F 20 400-419 17 17 17 26 26 26 F F F 21 420-439 18 18 18 27 27 27 F F F 22 440-459 20 20 20 31 31 31 F F F 23 460-479 20 20 20 31 31 31 F F F 24 480-499 13 13 13 21 21 21 F F F 25 500-519 20 20 20 30 30 30 F F F 26 520-539 22 22 22 34 34 34 F F F 27 540-559 22 22 22 34 34 34 F F F 28 560-579 16 16 16 25 25 25 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 23, Max flush: 35 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.084913804Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.093045291Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.093322782Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.09335096Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.093360309Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.093372239Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229} {"msg":"Crucible cf6a767e-7843-471b-964f-280bad9d1945 has session id: a3b8509e-93d9-4a2f-82f2-86deb21ce1b8","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.093705586Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"Upstairs opts: Upstairs UUID: cf6a767e-7843-471b-964f-280bad9d1945, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.093744563Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"Crucible stats registered with UUID: cf6a767e-7843-471b-964f-280bad9d1945","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.093755822Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:41.093770351Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.093947019Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.093977307Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.093988537Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.094139686Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.094156965Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229} {"msg":"cf6a767e-7843-471b-964f-280bad9d1945 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.094181314Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.094193323Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"client":"0","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.094203402Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"client":"1","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.094213511Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"client":"2","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.094223501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.09423373Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.094244489Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:36308)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.094311725Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"io task","client":"1","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:62979)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.094325764Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"io task","client":"2","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:38891)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.094336483Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"io task","client":"0","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095000698Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"client":"0","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095037585Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"client":"2","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095070443Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"client":"1","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095351004Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095361163Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"[0]R flush_numbers[0..12]: [30, 34, 34, 35, 24, 34, 34, 34, 35, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095392051Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"downstairs","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"[0]R generation[0..12]: [20, 22, 22, 23, 15, 22, 22, 22, 23, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.09540639Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"downstairs","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095418129Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"downstairs","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"[1]R flush_numbers[0..12]: [30, 34, 34, 35, 24, 34, 34, 34, 35, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095428889Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"downstairs","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"[1]R generation[0..12]: [20, 22, 22, 23, 15, 22, 22, 22, 23, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095439628Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"downstairs","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095450787Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"downstairs","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"[2]R flush_numbers[0..12]: [30, 34, 34, 35, 24, 34, 34, 34, 35, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095461436Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"downstairs","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"[2]R generation[0..12]: [20, 22, 22, 23, 15, 22, 22, 22, 23, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095471956Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"downstairs","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095483165Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"downstairs","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095492894Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"downstairs","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095502934Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"downstairs","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095512483Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"downstairs","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095522312Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"":"downstairs","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095532302Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095541391Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"client":"0","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.09555131Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"client":"1","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.09556299Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"client":"2","":"downstairs","session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095573149Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095582768Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"cf6a767e-7843-471b-964f-280bad9d1945 is now active with session: a3b8509e-93d9-4a2f-82f2-86deb21ce1b8","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095593117Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095602557Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095610546Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"session_id":"a3b8509e-93d9-4a2f-82f2-86deb21ce1b8"} {"msg":"Activated sub_volume cf6a767e-7843-471b-964f-280bad9d1945","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095620016Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095628435Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.095905536Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"task":"crutest"} 01/30 Write block 484 len 36864 data: 3 4 4 4 4 4 4 2 2 02/30 Write block 72 len 8192 data: 3 3 03/30 Flush 04/30 Write block 413 len 16384 data: 2 2 3 3 05/30 Read block 30 len 8192 06/30 Write block 224 len 40960 data: 5 3 4 4 4 4 5 5 4 4 07/30 Read block 79 len 20480 08/30 Write block 136 len 28672 data: 2 3 3 3 3 4 4 09/30 Write block 294 len 28672 data: 2 2 2 2 4 3 4 10/30 Write block 499 len 36864 data: 2 2 2 3 3 3 3 2 2 11/30 Write block 124 len 20480 data: 2 3 3 3 3 12/30 Read block 81 len 24576 13/30 Read block 33 len 40960 14/30 Read block 269 len 8192 15/30 Read block 544 len 8192 16/30 Read block 329 len 8192 17/30 Write block 328 len 24576 data: 4 4 4 4 4 4 18/30 Write block 74 len 28672 data: 3 2 2 2 2 2 2 19/30 Read block 248 len 24576 20/30 Write block 46 len 4096 data: 5 21/30 Read block 568 len 32768 22/30 Read block 64 len 16384 23/30 Write block 478 len 16384 data: 2 2 2 2 24/30 Flush 25/30 Write block 276 len 4096 data: 2 26/30 Read block 351 len 28672 27/30 Write block 57 len 24576 data: 6 5 5 5 6 6 28/30 Write block 493 len 4096 data: 2 29/30 Write block 539 len 32768 data: 3 3 3 3 3 3 3 2 30/30 Read block 106 len 40960 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 4 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Write 10 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Write 7 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 2 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Read 8 Done Sent Done false 1021 Acked Read 4 Done Sent Done false 1022 Acked Write 4 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 27 0 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1002)) Some(JobId(1023)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:41.10423326Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:41.104529959Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1229,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 22 22 22 34 34 34 F F F 2 040-059 24 22 24 37 34 37 T F T <--- 3 060-079 24 24 24 37 36 37 T F T <--- 4 080-099 24 15 24 37 24 37 F F F <--- 5 100-119 22 22 22 34 34 34 F F F 6 120-139 24 22 24 37 34 37 F F F <--- 7 140-159 24 22 24 37 34 37 F F F <--- 8 160-179 23 23 23 35 35 35 F F F 9 180-199 16 16 16 25 25 25 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 24 16 24 37 25 37 F F F <--- 12 240-259 18 18 18 27 27 27 F F F 13 260-279 20 20 20 30 30 30 T F T <--- 14 280-299 24 19 24 37 28 37 F F F <--- 15 300-319 24 22 24 37 34 37 F F F <--- 16 320-339 24 21 24 37 32 37 F F F <--- 17 340-359 23 23 23 35 35 35 F F F 18 360-379 21 21 21 32 32 32 F F F 19 380-399 19 19 19 28 28 28 F F F 20 400-419 24 17 24 37 26 37 F F F <--- 21 420-439 18 18 18 27 27 27 F F F 22 440-459 20 20 20 31 31 31 F F F 23 460-479 24 20 24 37 31 37 F F F <--- 24 480-499 24 24 24 37 36 37 T F T <--- 25 500-519 24 20 24 37 30 37 F F F <--- 26 520-539 22 22 22 34 34 34 T F T <--- 27 540-559 22 22 22 34 34 34 T F T <--- 28 560-579 16 16 16 25 25 25 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 24, Max flush: 37 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 22 22 22 34 34 34 F F F 2 040-059 25 25 25 38 38 38 F F F 3 060-079 25 25 25 38 38 38 F F F 4 080-099 24 24 24 37 37 37 F F F 5 100-119 22 22 22 34 34 34 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 24 24 24 37 37 37 F F F 8 160-179 23 23 23 35 35 35 F F F 9 180-199 16 16 16 25 25 25 F F F 10 200-219 16 16 16 25 25 25 F F F 11 220-239 24 24 24 37 37 37 F F F 12 240-259 18 18 18 27 27 27 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 24 24 24 37 37 37 F F F 15 300-319 24 24 24 37 37 37 F F F 16 320-339 24 24 24 37 37 37 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 21 21 21 32 32 32 F F F 19 380-399 19 19 19 28 28 28 F F F 20 400-419 24 24 24 37 37 37 F F F 21 420-439 18 18 18 27 27 27 F F F 22 440-459 20 20 20 31 31 31 F F F 23 460-479 24 24 24 37 37 37 F F F 24 480-499 25 25 25 38 38 38 F F F 25 500-519 24 24 24 37 37 37 F F F 26 520-539 25 25 25 38 38 38 F F F 27 540-559 25 25 25 38 38 38 F F F 28 560-579 16 16 16 25 25 25 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 25, Max flush: 38 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.352930902Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.36105333Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.361334271Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.361364839Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.361374008Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.361385917Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237} {"msg":"Crucible e9fb16c8-0413-48e8-83df-be2279d1a410 has session id: 71a11a96-d36d-4122-a70e-f24ce3c65deb","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.361703476Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"Upstairs opts: Upstairs UUID: e9fb16c8-0413-48e8-83df-be2279d1a410, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.361724184Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"Crucible stats registered with UUID: e9fb16c8-0413-48e8-83df-be2279d1a410","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.361734624Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:45.361747963Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.361941259Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.361972297Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.361983617Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.362099549Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.362114828Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237} {"msg":"e9fb16c8-0413-48e8-83df-be2279d1a410 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.362138916Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.362148685Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"client":"0","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.362158715Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"client":"1","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.362168714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"client":"2","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.362178593Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.362189013Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.362243539Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"ds_connection connected from Ok(127.0.0.1:39652)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.362271917Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"io task","client":"2","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"ds_connection connected from Ok(127.0.0.1:52156)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.362302245Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"io task","client":"0","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48089)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.362341992Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"io task","client":"1","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363014326Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"client":"2","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363034835Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"client":"0","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363046274Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"client":"1","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363344264Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363354173Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"[0]R flush_numbers[0..12]: [30, 34, 38, 38, 37, 34, 37, 37, 35, 25, 25, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363393651Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"downstairs","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"[0]R generation[0..12]: [20, 22, 25, 25, 24, 22, 24, 24, 23, 16, 16, 24]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.36340507Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"downstairs","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363416819Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"downstairs","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"[1]R flush_numbers[0..12]: [30, 34, 38, 38, 37, 34, 37, 37, 35, 25, 25, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363427538Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"downstairs","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"[1]R generation[0..12]: [20, 22, 25, 25, 24, 22, 24, 24, 23, 16, 16, 24]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363438388Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"downstairs","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363449477Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"downstairs","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"[2]R flush_numbers[0..12]: [30, 34, 38, 38, 37, 34, 37, 37, 35, 25, 25, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363460066Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"downstairs","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"[2]R generation[0..12]: [20, 22, 25, 25, 24, 22, 24, 24, 23, 16, 16, 24]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363472985Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"downstairs","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363484264Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"downstairs","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363494064Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"downstairs","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363504013Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"downstairs","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363513662Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"downstairs","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363523412Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"":"downstairs","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363533701Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.36354279Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"client":"0","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.36355282Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"client":"1","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363562789Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"client":"2","":"downstairs","session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363572798Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363582168Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"e9fb16c8-0413-48e8-83df-be2279d1a410 is now active with session: 71a11a96-d36d-4122-a70e-f24ce3c65deb","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363592337Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363601836Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363610176Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"session_id":"71a11a96-d36d-4122-a70e-f24ce3c65deb"} {"msg":"Activated sub_volume e9fb16c8-0413-48e8-83df-be2279d1a410","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363619585Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363629395Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.363955022Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"task":"crutest"} 01/30 Write block 72 len 40960 data: 4 4 4 3 3 3 3 3 3 2 02/30 Read block 318 len 32768 03/30 Write block 532 len 8192 data: 4 4 04/30 Write block 169 len 32768 data: 5 6 5 4 4 4 4 3 05/30 Read block 245 len 40960 06/30 Write block 377 len 24576 data: 4 6 5 4 4 4 07/30 Write block 364 len 20480 data: 5 5 4 2 3 08/30 Read block 413 len 36864 09/30 Read block 345 len 8192 10/30 Read block 503 len 20480 11/30 Read block 487 len 28672 12/30 Read block 544 len 40960 13/30 Read block 17 len 8192 14/30 Write block 447 len 8192 data: 2 2 15/30 Flush 16/30 Write block 542 len 16384 data: 4 4 4 4 17/30 Write block 331 len 28672 data: 5 5 5 3 3 4 4 18/30 Read block 334 len 8192 19/30 Write block 401 len 32768 data: 3 4 3 3 3 3 3 3 20/30 Read block 204 len 28672 21/30 Write block 92 len 12288 data: 3 4 3 22/30 Read block 72 len 28672 23/30 Read block 217 len 8192 24/30 Write block 27 len 8192 data: 3 3 25/30 Write block 63 len 4096 data: 4 26/30 Write block 198 len 28672 data: 2 2 2 2 2 2 4 27/30 Read block 297 len 32768 28/30 Write block 296 len 16384 data: 3 3 5 4 29/30 Read block 220 len 32768 30/30 Read block 16 len 12288 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 8 Done Sent Done false 1002 Acked Write 2 Done Sent Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 4 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 7 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 30 1 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:45.372477233Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:45.372770263Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1237,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 22 22 22 34 34 34 T F T <--- 2 040-059 25 25 25 38 38 38 F F F 3 060-079 26 25 26 39 38 39 T T T <--- 4 080-099 26 24 26 39 37 39 T T T <--- 5 100-119 22 22 22 34 34 34 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 24 24 24 37 37 37 F F F 8 160-179 26 23 26 39 35 39 F F F <--- 9 180-199 16 16 16 25 25 25 T F T <--- 10 200-219 16 16 16 25 25 25 T F T <--- 11 220-239 24 24 24 37 37 37 F F F 12 240-259 18 18 18 27 27 27 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 24 24 24 37 37 37 T F T <--- 15 300-319 24 24 24 37 37 37 F F F 16 320-339 24 24 24 37 37 37 T F T <--- 17 340-359 23 23 23 35 35 35 F F F 18 360-379 26 21 26 39 32 39 F F F <--- 19 380-399 26 19 26 39 28 39 F F F <--- 20 400-419 24 24 24 37 37 37 T F T <--- 21 420-439 18 18 18 27 27 27 F F F 22 440-459 26 20 26 39 31 39 F F F <--- 23 460-479 24 24 24 37 37 37 F F F 24 480-499 25 25 25 38 38 38 F F F 25 500-519 24 24 24 37 37 37 F F F 26 520-539 26 25 26 39 38 39 F F F <--- 27 540-559 25 25 25 38 38 38 T F T <--- 28 560-579 16 16 16 25 25 25 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 26, Max flush: 39 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 27 27 27 40 40 40 F F F 2 040-059 25 25 25 38 38 38 F F F 3 060-079 27 27 27 40 40 40 F F F 4 080-099 27 27 27 40 40 40 F F F 5 100-119 22 22 22 34 34 34 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 24 24 24 37 37 37 F F F 8 160-179 26 26 26 39 39 39 F F F 9 180-199 27 27 27 40 40 40 F F F 10 200-219 27 27 27 40 40 40 F F F 11 220-239 24 24 24 37 37 37 F F F 12 240-259 18 18 18 27 27 27 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 27 27 27 40 40 40 F F F 15 300-319 24 24 24 37 37 37 F F F 16 320-339 27 27 27 40 40 40 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 26 26 26 39 39 39 F F F 19 380-399 26 26 26 39 39 39 F F F 20 400-419 27 27 27 40 40 40 F F F 21 420-439 18 18 18 27 27 27 F F F 22 440-459 26 26 26 39 39 39 F F F 23 460-479 24 24 24 37 37 37 F F F 24 480-499 25 25 25 38 38 38 F F F 25 500-519 24 24 24 37 37 37 F F F 26 520-539 26 26 26 39 39 39 F F F 27 540-559 27 27 27 40 40 40 F F F 28 560-579 16 16 16 25 25 25 F F F 29 580-599 22 22 22 34 34 34 F F F Max gen: 27, Max flush: 40 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.619599893Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:49.626816492Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.626882647Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.633810836Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634065988Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634089737Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634098716Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634110575Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245} {"msg":"Crucible e21091d2-157b-4846-9a3c-f1315d75a08c has session id: d7f1a870-b7ca-455e-9c6b-bf3765c188d9","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634406335Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"Upstairs opts: Upstairs UUID: e21091d2-157b-4846-9a3c-f1315d75a08c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634427624Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"Crucible stats registered with UUID: e21091d2-157b-4846-9a3c-f1315d75a08c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634438143Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:49.634451002Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.63462611Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634645809Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634656458Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634790609Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634815087Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245} {"msg":"e21091d2-157b-4846-9a3c-f1315d75a08c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634824357Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634834086Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"client":"0","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634844735Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"client":"1","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634854995Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"client":"2","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634864934Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634875533Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634886033Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33752)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634960238Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"io task","client":"1","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55976)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.634986866Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"io task","client":"0","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"ds_connection connected from Ok(127.0.0.1:57111)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.635003335Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"io task","client":"2","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.635578585Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"client":"2","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.635596194Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"client":"1","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.635618383Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"client":"0","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.635882685Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.635893884Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"[0]R flush_numbers[0..12]: [30, 40, 38, 40, 40, 34, 37, 37, 39, 40, 40, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.635928782Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"downstairs","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"[0]R generation[0..12]: [20, 27, 25, 27, 27, 22, 24, 24, 26, 27, 27, 24]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.635940291Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"downstairs","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.63595168Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"downstairs","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"[1]R flush_numbers[0..12]: [30, 40, 38, 40, 40, 34, 37, 37, 39, 40, 40, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.635962749Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"downstairs","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"[1]R generation[0..12]: [20, 27, 25, 27, 27, 22, 24, 24, 26, 27, 27, 24]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.635973659Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"downstairs","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.635985108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"downstairs","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"[2]R flush_numbers[0..12]: [30, 40, 38, 40, 40, 34, 37, 37, 39, 40, 40, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.635995957Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"downstairs","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"[2]R generation[0..12]: [20, 27, 25, 27, 27, 22, 24, 24, 26, 27, 27, 24]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636017356Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"downstairs","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636028875Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"downstairs","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636038734Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"downstairs","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636048923Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"downstairs","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636060583Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"downstairs","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636070442Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"":"downstairs","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636080671Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636089701Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"client":"0","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.63609976Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"client":"1","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636109819Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"client":"2","":"downstairs","session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636120119Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636129618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"e21091d2-157b-4846-9a3c-f1315d75a08c is now active with session: d7f1a870-b7ca-455e-9c6b-bf3765c188d9","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636139897Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636149417Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636157316Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"session_id":"d7f1a870-b7ca-455e-9c6b-bf3765c188d9"} {"msg":"Activated sub_volume e21091d2-157b-4846-9a3c-f1315d75a08c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636166875Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636175355Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.636433837Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"task":"crutest"} 01/30 Write block 76 len 28672 data: 4 4 4 4 4 3 2 02/30 Write block 77 len 8192 data: 5 5 03/30 Read block 235 len 32768 04/30 Read block 370 len 36864 05/30 Write block 168 len 28672 data: 6 6 7 6 5 5 5 06/30 Read block 553 len 20480 07/30 Read block 480 len 20480 08/30 Flush 09/30 Write block 39 len 24576 data: 5 5 5 5 4 5 10/30 Write block 463 len 24576 data: 5 6 6 6 2 2 11/30 Write block 416 len 32768 data: 4 3 3 3 3 3 3 3 12/30 Write block 169 len 24576 data: 7 8 7 6 6 6 13/30 Read block 379 len 12288 14/30 Write block 561 len 4096 data: 3 15/30 Read block 354 len 36864 16/30 Flush 17/30 Write block 295 len 32768 data: 3 4 4 6 5 5 4 4 18/30 Read block 276 len 32768 19/30 Read block 200 len 4096 20/30 Write block 322 len 36864 data: 4 3 3 2 2 3 5 5 5 21/30 Flush 22/30 Write block 478 len 12288 data: 3 3 3 23/30 Read block 167 len 20480 24/30 Read block 112 len 12288 25/30 Read block 60 len 20480 26/30 Read block 328 len 16384 27/30 Write block 415 len 16384 data: 4 5 4 4 28/30 Read block 285 len 36864 29/30 Write block 580 len 32768 data: 4 4 3 2 2 3 3 3 30/30 Read block 12 len 28672 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 2 Sent Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Read 9 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Write 1 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Write 8 Sent Done Sent false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 2 31 Done 1 30 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:49.645831888Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:49.646129598Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1245,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 27 28 28 40 42 42 F F F <--- 2 040-059 25 28 28 38 42 42 F F F <--- 3 060-079 27 28 28 40 41 41 T F F <--- 4 080-099 27 28 28 40 41 41 T F F <--- 5 100-119 22 22 22 34 34 34 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 24 24 24 37 37 37 F F F 8 160-179 26 28 28 39 42 42 F F F <--- 9 180-199 27 27 27 40 40 40 F F F 10 200-219 27 27 27 40 40 40 F F F 11 220-239 24 24 24 37 37 37 F F F 12 240-259 18 18 18 27 27 27 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 27 28 28 40 43 43 F F F <--- 15 300-319 24 28 28 37 43 43 F F F <--- 16 320-339 27 28 28 40 43 43 F F F <--- 17 340-359 23 23 23 35 35 35 F F F 18 360-379 26 26 26 39 39 39 F F F 19 380-399 26 26 26 39 39 39 F F F 20 400-419 27 28 28 40 42 42 F T T <--- 21 420-439 18 28 28 27 42 42 F F F <--- 22 440-459 26 26 26 39 39 39 F F F 23 460-479 24 28 28 37 42 42 F T T <--- 24 480-499 25 25 25 38 38 38 F T T <--- 25 500-519 24 24 24 37 37 37 F F F 26 520-539 26 26 26 39 39 39 F F F 27 540-559 27 27 27 40 40 40 F F F 28 560-579 16 28 28 25 42 42 F F F <--- 29 580-599 22 22 22 34 34 34 F T T <--- Max gen: 28, Max flush: 43 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 28 28 28 42 42 42 F F F 2 040-059 28 28 28 42 42 42 F F F 3 060-079 28 28 28 41 41 41 F F F 4 080-099 28 28 28 41 41 41 F F F 5 100-119 22 22 22 34 34 34 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 24 24 24 37 37 37 F F F 8 160-179 28 28 28 42 42 42 F F F 9 180-199 27 27 27 40 40 40 F F F 10 200-219 27 27 27 40 40 40 F F F 11 220-239 24 24 24 37 37 37 F F F 12 240-259 18 18 18 27 27 27 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 28 28 28 43 43 43 F F F 15 300-319 28 28 28 43 43 43 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 26 26 26 39 39 39 F F F 19 380-399 26 26 26 39 39 39 F F F 20 400-419 29 29 29 44 44 44 F F F 21 420-439 28 28 28 42 42 42 F F F 22 440-459 26 26 26 39 39 39 F F F 23 460-479 29 29 29 44 44 44 F F F 24 480-499 29 29 29 44 44 44 F F F 25 500-519 24 24 24 37 37 37 F F F 26 520-539 26 26 26 39 39 39 F F F 27 540-559 27 27 27 40 40 40 F F F 28 560-579 28 28 28 42 42 42 F F F 29 580-599 29 29 29 44 44 44 F F F Max gen: 29, Max flush: 44 Begin loop 15 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.89540774Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:53.902705803Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.902772199Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.909530129Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.909785502Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.90980862Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.909817369Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.909829199Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253} {"msg":"Crucible 9beb54ec-a273-45bb-a734-1c2ac3fce6dd has session id: f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910179445Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"Upstairs opts: Upstairs UUID: 9beb54ec-a273-45bb-a734-1c2ac3fce6dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910211553Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"Crucible stats registered with UUID: 9beb54ec-a273-45bb-a734-1c2ac3fce6dd","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910222772Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:53.910232391Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.91039321Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910425538Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910436897Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.91054487Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910558279Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253} {"msg":"9beb54ec-a273-45bb-a734-1c2ac3fce6dd active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910593787Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910606126Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"client":"0","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910616285Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"client":"1","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910626394Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"client":"2","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910636424Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910646783Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910656962Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"ds_connection connected from Ok(127.0.0.1:61654)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910713788Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"io task","client":"2","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"ds_connection connected from Ok(127.0.0.1:52006)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910733797Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"io task","client":"0","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"ds_connection connected from Ok(127.0.0.1:52609)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.910829021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"io task","client":"1","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911232373Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"client":"2","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911320677Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"client":"1","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911478276Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"client":"0","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911531473Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911599888Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"[0]R flush_numbers[0..12]: [30, 42, 42, 41, 41, 34, 37, 37, 42, 40, 40, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911772586Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"downstairs","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"[0]R generation[0..12]: [20, 28, 28, 28, 28, 22, 24, 24, 28, 27, 27, 24]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911786355Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"downstairs","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911798005Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"downstairs","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"[1]R flush_numbers[0..12]: [30, 42, 42, 41, 41, 34, 37, 37, 42, 40, 40, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911808834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"downstairs","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"[1]R generation[0..12]: [20, 28, 28, 28, 28, 22, 24, 24, 28, 27, 27, 24]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911819523Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"downstairs","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911830792Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"downstairs","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"[2]R flush_numbers[0..12]: [30, 42, 42, 41, 41, 34, 37, 37, 42, 40, 40, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911841552Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"downstairs","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"[2]R generation[0..12]: [20, 28, 28, 28, 28, 22, 24, 24, 28, 27, 27, 24]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911852341Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"downstairs","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.91186358Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"downstairs","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.91187365Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"downstairs","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911883859Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"downstairs","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911893758Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"downstairs","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911903577Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"":"downstairs","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911913977Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911923296Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"client":"0","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911934585Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"client":"1","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911944975Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"client":"2","":"downstairs","session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911955394Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911964973Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"9beb54ec-a273-45bb-a734-1c2ac3fce6dd is now active with session: f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911975273Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911984822Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.911992711Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"session_id":"f07f2dea-1ec4-48e5-ba51-fd6d3ec63d58"} {"msg":"Activated sub_volume 9beb54ec-a273-45bb-a734-1c2ac3fce6dd","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.912002181Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.91201082Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.912362256Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"task":"crutest"} 01/30 Write block 447 len 32768 data: 3 3 2 2 2 3 3 5 02/30 Read block 422 len 8192 03/30 Flush 04/30 Flush 05/30 Write block 520 len 32768 data: 2 2 2 2 2 3 4 4 06/30 Write block 318 len 8192 data: 4 4 07/30 Read block 382 len 28672 08/30 Write block 109 len 32768 data: 5 5 6 4 4 5 4 4 09/30 Read block 507 len 4096 10/30 Write block 507 len 40960 data: 3 2 3 3 3 3 3 3 3 3 11/30 Read block 266 len 36864 12/30 Write block 463 len 36864 data: 6 7 7 7 3 3 2 2 2 13/30 Read block 503 len 8192 14/30 Read block 119 len 8192 15/30 Write block 369 len 16384 data: 3 2 2 3 16/30 Write block 445 len 24576 data: 2 2 4 4 3 3 17/30 Write block 404 len 20480 data: 4 4 4 4 4 18/30 Write block 414 len 24576 data: 3 5 6 5 5 4 19/30 Read block 253 len 32768 20/30 Write block 569 len 40960 data: 4 4 4 4 5 5 7 6 5 5 21/30 Read block 160 len 20480 22/30 Flush 23/30 Write block 80 len 4096 data: 5 24/30 Read block 134 len 8192 25/30 Write block 394 len 24576 data: 4 4 4 4 4 4 26/30 Write block 238 len 36864 data: 3 3 3 4 5 5 5 6 6 27/30 Read block 485 len 12288 28/30 Read block 484 len 40960 29/30 Write block 69 len 36864 data: 3 2 3 5 5 5 4 5 6 30/30 Write block 546 len 12288 data: 3 3 3 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 8 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Write 8 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 13 2 2 17 Done 13 24 24 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:53.92113146Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:53.921400461Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1253,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 28 28 28 42 42 42 F F F 2 040-059 28 28 28 42 42 42 F F F 3 060-079 28 28 28 41 41 41 F T T <--- 4 080-099 28 28 28 41 41 41 F T T <--- 5 100-119 22 30 30 34 47 47 T F F <--- 6 120-139 24 24 24 37 37 37 F F F 7 140-159 24 24 24 37 37 37 F F F 8 160-179 28 28 28 42 42 42 F F F 9 180-199 27 27 27 40 40 40 F F F 10 200-219 27 27 27 40 40 40 F F F 11 220-239 24 24 24 37 37 37 F T T <--- 12 240-259 18 18 18 27 27 27 F T T <--- 13 260-279 25 25 25 38 38 38 F F F 14 280-299 28 28 28 43 43 43 F F F 15 300-319 28 30 30 43 47 47 T F F <--- 16 320-339 28 28 28 43 43 43 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 26 30 30 39 47 47 T F F <--- 19 380-399 26 26 26 39 39 39 F T T <--- 20 400-419 29 30 30 44 47 47 T F F <--- 21 420-439 28 28 28 42 42 42 F F F 22 440-459 30 30 30 45 47 47 T F F <--- 23 460-479 29 30 30 44 47 47 T F F <--- 24 480-499 29 29 29 44 44 44 F F F 25 500-519 24 30 30 37 47 47 T F F <--- 26 520-539 26 30 30 39 47 47 T F F <--- 27 540-559 27 27 27 40 40 40 F T T <--- 28 560-579 28 30 30 42 47 47 F F F <--- 29 580-599 29 29 29 44 44 44 F F F Max gen: 30, Max flush: 47 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 28 28 28 42 42 42 F F F 2 040-059 28 28 28 42 42 42 F F F 3 060-079 31 31 31 48 48 48 F F F 4 080-099 31 31 31 48 48 48 F F F 5 100-119 30 30 30 47 47 47 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 24 24 24 37 37 37 F F F 8 160-179 28 28 28 42 42 42 F F F 9 180-199 27 27 27 40 40 40 F F F 10 200-219 27 27 27 40 40 40 F F F 11 220-239 31 31 31 48 48 48 F F F 12 240-259 31 31 31 48 48 48 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 28 28 28 43 43 43 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 30 30 30 47 47 47 F F F 19 380-399 31 31 31 48 48 48 F F F 20 400-419 30 30 30 47 47 47 F F F 21 420-439 28 28 28 42 42 42 F F F 22 440-459 30 30 30 47 47 47 F F F 23 460-479 30 30 30 47 47 47 F F F 24 480-499 29 29 29 44 44 44 F F F 25 500-519 30 30 30 47 47 47 F F F 26 520-539 30 30 30 47 47 47 F F F 27 540-559 31 31 31 48 48 48 F F F 28 560-579 30 30 30 47 47 47 F F F 29 580-599 29 29 29 44 44 44 F F F Max gen: 31, Max flush: 48 Begin loop 16 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.175539494Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:58.182618362Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.182679118Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.189465896Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.189717959Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.189745097Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.189753837Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.189768386Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261} {"msg":"Crucible 8847db1d-257d-406f-8d58-8111f53d4f89 has session id: 60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190063826Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"Upstairs opts: Upstairs UUID: 8847db1d-257d-406f-8d58-8111f53d4f89, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190095694Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"Crucible stats registered with UUID: 8847db1d-257d-406f-8d58-8111f53d4f89","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190107223Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:58.190116842Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190276031Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190308279Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190319558Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190415342Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190430181Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261} {"msg":"8847db1d-257d-406f-8d58-8111f53d4f89 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.19044871Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190458279Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"client":"0","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190468468Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"client":"1","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190478548Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"client":"2","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190488507Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190498906Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190511005Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"ds_connection connected from Ok(127.0.0.1:65205)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190560002Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"io task","client":"2","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"ds_connection connected from Ok(127.0.0.1:64915)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.19059232Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"io task","client":"1","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"ds_connection connected from Ok(127.0.0.1:62762)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.190603149Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"io task","client":"0","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191308941Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"client":"1","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191374477Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"client":"0","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191393785Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"client":"2","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191653908Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191693545Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"[0]R flush_numbers[0..12]: [30, 42, 42, 48, 48, 47, 37, 37, 42, 40, 40, 48]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191730142Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"downstairs","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"[0]R generation[0..12]: [20, 28, 28, 31, 31, 30, 24, 24, 28, 27, 27, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191741362Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"downstairs","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191752671Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"downstairs","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"[1]R flush_numbers[0..12]: [30, 42, 42, 48, 48, 47, 37, 37, 42, 40, 40, 48]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.19176347Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"downstairs","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"[1]R generation[0..12]: [20, 28, 28, 31, 31, 30, 24, 24, 28, 27, 27, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191774329Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"downstairs","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191787819Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"downstairs","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"[2]R flush_numbers[0..12]: [30, 42, 42, 48, 48, 47, 37, 37, 42, 40, 40, 48]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191798638Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"downstairs","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"[2]R generation[0..12]: [20, 28, 28, 31, 31, 30, 24, 24, 28, 27, 27, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191809317Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"downstairs","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191820506Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"downstairs","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191830086Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"downstairs","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191839965Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"downstairs","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191849624Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"downstairs","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191859194Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"":"downstairs","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191869263Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191878422Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"client":"0","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191888612Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"client":"1","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191898611Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"client":"2","":"downstairs","session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.19190867Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.19191822Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"8847db1d-257d-406f-8d58-8111f53d4f89 is now active with session: 60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191928559Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191939168Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191947108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"session_id":"60e5d2ba-1f4b-4c42-b0f3-cb7ce1b1dda4"} {"msg":"Activated sub_volume 8847db1d-257d-406f-8d58-8111f53d4f89","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191956527Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.191965036Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.192242918Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"task":"crutest"} 01/30 Write block 36 len 32768 data: 3 3 4 6 6 6 6 5 02/30 Read block 115 len 16384 03/30 Read block 391 len 16384 04/30 Read block 581 len 36864 05/30 Read block 166 len 20480 06/30 Read block 71 len 12288 07/30 Read block 509 len 8192 08/30 Write block 507 len 20480 data: 4 3 4 4 4 09/30 Read block 426 len 8192 10/30 Read block 336 len 8192 11/30 Flush 12/30 Write block 544 len 20480 data: 5 5 4 4 4 13/30 Read block 101 len 8192 14/30 Read block 389 len 24576 15/30 Read block 333 len 8192 16/30 Read block 182 len 16384 17/30 Write block 369 len 8192 data: 4 3 18/30 Read block 573 len 32768 19/30 Read block 332 len 12288 20/30 Read block 87 len 8192 21/30 Read block 387 len 12288 22/30 Read block 447 len 24576 23/30 Write block 397 len 36864 data: 5 5 5 3 4 5 4 5 5 24/30 Read block 317 len 12288 25/30 Write block 33 len 16384 data: 3 3 3 4 26/30 Write block 410 len 24576 data: 2 2 2 3 4 6 27/30 Read block 420 len 4096 28/30 Write block 28 len 24576 data: 4 3 3 3 2 4 29/30 Write block 92 len 32768 data: 4 5 4 4 4 5 5 5 30/30 Read block 1 len 12288 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Read 4 Sent Done Done false 1003 Acked Read 9 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Write 5 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Write 8 Sent Done Sent false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 2 30 Done 2 30 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:39:58.200352546Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:39:58.200639476Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1261,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 28 32 32 42 49 49 T T T <--- 2 040-059 28 32 32 42 49 49 T F F <--- 3 060-079 31 31 31 48 48 48 F F F 4 080-099 31 31 31 48 48 48 F T T <--- 5 100-119 30 30 30 47 47 47 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 24 24 24 37 37 37 F F F 8 160-179 28 28 28 42 42 42 F F F 9 180-199 27 27 27 40 40 40 F F F 10 200-219 27 27 27 40 40 40 F F F 11 220-239 31 31 31 48 48 48 F F F 12 240-259 31 31 31 48 48 48 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 28 28 28 43 43 43 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 30 30 30 47 47 47 F T T <--- 19 380-399 31 31 31 48 48 48 F T T <--- 20 400-419 30 30 30 47 47 47 F T T <--- 21 420-439 28 28 28 42 42 42 F F F 22 440-459 30 30 30 47 47 47 F F F 23 460-479 30 30 30 47 47 47 F F F 24 480-499 29 29 29 44 44 44 F F F 25 500-519 30 32 32 47 49 49 F F F <--- 26 520-539 30 30 30 47 47 47 F F F 27 540-559 31 31 31 48 48 48 F T T <--- 28 560-579 30 30 30 47 47 47 F F F 29 580-599 29 29 29 44 44 44 F F F Max gen: 32, Max flush: 49 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 33 33 33 50 50 50 F F F 2 040-059 32 32 32 49 49 49 F F F 3 060-079 31 31 31 48 48 48 F F F 4 080-099 33 33 33 50 50 50 F F F 5 100-119 30 30 30 47 47 47 F F F 6 120-139 24 24 24 37 37 37 F F F 7 140-159 24 24 24 37 37 37 F F F 8 160-179 28 28 28 42 42 42 F F F 9 180-199 27 27 27 40 40 40 F F F 10 200-219 27 27 27 40 40 40 F F F 11 220-239 31 31 31 48 48 48 F F F 12 240-259 31 31 31 48 48 48 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 28 28 28 43 43 43 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 33 33 33 50 50 50 F F F 19 380-399 33 33 33 50 50 50 F F F 20 400-419 33 33 33 50 50 50 F F F 21 420-439 28 28 28 42 42 42 F F F 22 440-459 30 30 30 47 47 47 F F F 23 460-479 30 30 30 47 47 47 F F F 24 480-499 29 29 29 44 44 44 F F F 25 500-519 32 32 32 49 49 49 F F F 26 520-539 30 30 30 47 47 47 F F F 27 540-559 33 33 33 50 50 50 F F F 28 560-579 30 30 30 47 47 47 F F F 29 580-599 29 29 29 44 44 44 F F F Max gen: 33, Max flush: 50 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.440187202Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.448546583Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.448818704Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.448848452Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.448857492Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.448869711Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269} {"msg":"Crucible 4a18ff54-648c-4737-9435-3b510937674a has session id: ab8bc31b-3271-4343-bf2b-4ef72c08d546","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449270134Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"Upstairs opts: Upstairs UUID: 4a18ff54-648c-4737-9435-3b510937674a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449310931Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"Crucible stats registered with UUID: 4a18ff54-648c-4737-9435-3b510937674a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.44932255Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:02.449332789Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449489499Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449509287Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449519927Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449656297Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449679486Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269} {"msg":"4a18ff54-648c-4737-9435-3b510937674a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449697155Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449706434Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"client":"0","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449716503Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"client":"1","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449726453Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"client":"2","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449736382Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449746751Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449757001Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"ds_connection connected from Ok(127.0.0.1:56731)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449812467Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"io task","client":"2","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"ds_connection connected from Ok(127.0.0.1:64592)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449839435Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"io task","client":"0","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"ds_connection connected from Ok(127.0.0.1:37556)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.449851004Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"io task","client":"1","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.450527928Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"client":"0","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.450545867Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"client":"2","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.450560526Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"client":"1","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.450825688Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.450857906Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"[0]R flush_numbers[0..12]: [30, 50, 49, 48, 50, 47, 37, 37, 42, 40, 40, 48]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.450987507Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"downstairs","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"[0]R generation[0..12]: [20, 33, 32, 31, 33, 30, 24, 24, 28, 27, 27, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.450999066Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"downstairs","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451010665Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"downstairs","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"[1]R flush_numbers[0..12]: [30, 50, 49, 48, 50, 47, 37, 37, 42, 40, 40, 48]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451021395Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"downstairs","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"[1]R generation[0..12]: [20, 33, 32, 31, 33, 30, 24, 24, 28, 27, 27, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451032164Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"downstairs","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451043383Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"downstairs","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"[2]R flush_numbers[0..12]: [30, 50, 49, 48, 50, 47, 37, 37, 42, 40, 40, 48]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451054072Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"downstairs","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"[2]R generation[0..12]: [20, 33, 32, 31, 33, 30, 24, 24, 28, 27, 27, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451064742Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"downstairs","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451075901Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"downstairs","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.45108585Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"downstairs","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451096189Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"downstairs","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451107319Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"downstairs","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451117138Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"":"downstairs","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451127297Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451136547Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"client":"0","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451146636Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"client":"1","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451156595Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"client":"2","":"downstairs","session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451166875Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451176504Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"4a18ff54-648c-4737-9435-3b510937674a is now active with session: ab8bc31b-3271-4343-bf2b-4ef72c08d546","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451186823Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451196303Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451204222Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"session_id":"ab8bc31b-3271-4343-bf2b-4ef72c08d546"} {"msg":"Activated sub_volume 4a18ff54-648c-4737-9435-3b510937674a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451213681Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451222111Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.451506702Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"task":"crutest"} 01/30 Write block 491 len 28672 data: 3 3 3 2 2 3 3 02/30 Flush 03/30 Write block 107 len 8192 data: 4 6 04/30 Flush 05/30 Write block 478 len 28672 data: 4 4 4 3 3 3 4 06/30 Read block 491 len 40960 07/30 Write block 467 len 36864 data: 4 4 3 3 3 2 2 2 2 08/30 Read block 43 len 28672 09/30 Write block 562 len 36864 data: 3 3 3 3 3 3 4 5 5 10/30 Flush 11/30 Flush 12/30 Write block 181 len 36864 data: 2 2 2 3 5 4 4 5 5 13/30 Read block 406 len 40960 14/30 Write block 365 len 24576 data: 6 5 3 4 5 4 15/30 Read block 55 len 28672 16/30 Write block 437 len 24576 data: 5 5 4 4 3 3 17/30 Write block 537 len 16384 data: 4 4 4 4 18/30 Flush 19/30 Write block 427 len 40960 data: 4 4 4 4 6 6 6 6 7 6 20/30 Flush 21/30 Write block 121 len 24576 data: 2 2 2 3 4 4 22/30 Read block 141 len 8192 23/30 Flush 24/30 Write block 486 len 8192 data: 5 5 25/30 Write block 31 len 4096 data: 4 26/30 Read block 347 len 20480 27/30 Read block 424 len 8192 28/30 Read block 81 len 36864 29/30 Write block 567 len 16384 data: 4 5 6 6 30/30 Read block 203 len 16384 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 7 Done Done Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 25 0 25 Done 26 1 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1003)) Some(JobId(1022)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:02.461096359Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:02.461391279Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1269,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 33 33 33 50 50 50 T F T <--- 2 040-059 32 32 32 49 49 49 F F F 3 060-079 31 31 31 48 48 48 F F F 4 080-099 33 33 33 50 50 50 F F F 5 100-119 34 34 34 52 52 52 F F F 6 120-139 34 24 34 57 37 57 F F F <--- 7 140-159 24 24 24 37 37 37 F F F 8 160-179 28 28 28 42 42 42 F F F 9 180-199 34 27 34 55 40 55 F F F <--- 10 200-219 27 27 27 40 40 40 F F F 11 220-239 31 31 31 48 48 48 F F F 12 240-259 31 31 31 48 48 48 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 28 28 28 43 43 43 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 34 33 34 55 50 55 F F F <--- 19 380-399 33 33 33 50 50 50 F F F 20 400-419 33 33 33 50 50 50 F F F 21 420-439 34 28 34 56 42 56 F F F <--- 22 440-459 34 30 34 55 47 55 F F F <--- 23 460-479 34 30 34 53 47 53 F T F <--- 24 480-499 34 34 34 53 51 53 T T T <--- 25 500-519 32 32 32 49 49 49 F F F 26 520-539 34 30 34 55 47 55 F F F <--- 27 540-559 34 33 34 55 50 55 F F F <--- 28 560-579 34 30 34 53 47 53 T F T <--- 29 580-599 29 29 29 44 44 44 F F F Max gen: 34, Max flush: 57 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 32 32 32 49 49 49 F F F 3 060-079 31 31 31 48 48 48 F F F 4 080-099 33 33 33 50 50 50 F F F 5 100-119 34 34 34 52 52 52 F F F 6 120-139 34 34 34 57 57 57 F F F 7 140-159 24 24 24 37 37 37 F F F 8 160-179 28 28 28 42 42 42 F F F 9 180-199 34 34 34 55 55 55 F F F 10 200-219 27 27 27 40 40 40 F F F 11 220-239 31 31 31 48 48 48 F F F 12 240-259 31 31 31 48 48 48 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 28 28 28 43 43 43 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 28 28 28 43 43 43 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 34 34 34 55 55 55 F F F 19 380-399 33 33 33 50 50 50 F F F 20 400-419 33 33 33 50 50 50 F F F 21 420-439 34 34 34 56 56 56 F F F 22 440-459 34 34 34 55 55 55 F F F 23 460-479 34 34 34 53 53 53 F F F 24 480-499 35 35 35 58 58 58 F F F 25 500-519 32 32 32 49 49 49 F F F 26 520-539 34 34 34 55 55 55 F F F 27 540-559 34 34 34 55 55 55 F F F 28 560-579 35 35 35 58 58 58 F F F 29 580-599 29 29 29 44 44 44 F F F Max gen: 35, Max flush: 58 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.699755685Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.707978225Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.708288994Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.708315463Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.708326832Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.708339011Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277} {"msg":"Crucible b04e04a8-2e51-44a7-8697-510d4d91476a has session id: 13ddb224-7e1f-4eb4-b64d-465c8b55c5d7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.708679188Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"Upstairs opts: Upstairs UUID: b04e04a8-2e51-44a7-8697-510d4d91476a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.708710506Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"Crucible stats registered with UUID: b04e04a8-2e51-44a7-8697-510d4d91476a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.708722105Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:06.708741584Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.708929921Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.70894659Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.708957779Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.709103059Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.709116688Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277} {"msg":"b04e04a8-2e51-44a7-8697-510d4d91476a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.709133687Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.709143006Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"client":"0","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.709153156Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"client":"1","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.709163275Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"client":"2","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.709175644Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.709186023Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.709226501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:44274)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.709289056Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"io task","client":"2","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:35987)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.709300765Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"io task","client":"1","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"ds_connection connected from Ok(127.0.0.1:44075)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.709311695Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"io task","client":"0","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710072163Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"client":"1","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710094891Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"client":"2","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710129189Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"client":"0","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.71040788Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710432668Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"[0]R flush_numbers[0..12]: [30, 58, 49, 48, 50, 52, 57, 37, 42, 55, 40, 48]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710443768Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"downstairs","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"[0]R generation[0..12]: [20, 35, 32, 31, 33, 34, 34, 24, 28, 34, 27, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710455197Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"downstairs","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710466746Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"downstairs","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"[1]R flush_numbers[0..12]: [30, 58, 49, 48, 50, 52, 57, 37, 42, 55, 40, 48]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710477735Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"downstairs","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"[1]R generation[0..12]: [20, 35, 32, 31, 33, 34, 34, 24, 28, 34, 27, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710491175Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"downstairs","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710502654Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"downstairs","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"[2]R flush_numbers[0..12]: [30, 58, 49, 48, 50, 52, 57, 37, 42, 55, 40, 48]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710513583Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"downstairs","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"[2]R generation[0..12]: [20, 35, 32, 31, 33, 34, 34, 24, 28, 34, 27, 31]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710524332Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"downstairs","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710535781Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"downstairs","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710545581Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"downstairs","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.71055568Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"downstairs","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710565609Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"downstairs","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710575559Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"":"downstairs","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710585778Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710594757Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"client":"0","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710604987Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"client":"1","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710614986Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"client":"2","":"downstairs","session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710625435Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710634985Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"b04e04a8-2e51-44a7-8697-510d4d91476a is now active with session: 13ddb224-7e1f-4eb4-b64d-465c8b55c5d7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710646714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710656163Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710664163Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"session_id":"13ddb224-7e1f-4eb4-b64d-465c8b55c5d7"} {"msg":"Activated sub_volume b04e04a8-2e51-44a7-8697-510d4d91476a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710673702Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.710682092Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.711037217Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"task":"crutest"} 01/30 Write block 493 len 4096 data: 4 02/30 Read block 180 len 36864 03/30 Flush 04/30 Write block 134 len 40960 data: 3 2 3 4 4 4 4 5 5 4 05/30 Flush 06/30 Write block 425 len 12288 data: 4 4 5 07/30 Write block 501 len 32768 data: 3 4 4 4 4 3 5 4 08/30 Read block 523 len 28672 09/30 Flush 10/30 Write block 232 len 4096 data: 5 11/30 Read block 472 len 40960 12/30 Read block 405 len 16384 13/30 Read block 1 len 20480 14/30 Read block 50 len 24576 15/30 Read block 97 len 28672 16/30 Flush 17/30 Flush 18/30 Write block 469 len 8192 data: 4 4 19/30 Write block 587 len 20480 data: 4 2 2 2 3 20/30 Write block 392 len 40960 data: 3 3 5 5 5 6 6 6 4 5 21/30 Read block 61 len 24576 22/30 Write block 40 len 8192 data: 7 7 23/30 Read block 335 len 32768 24/30 Read block 360 len 12288 25/30 Write block 222 len 12288 data: 5 4 6 26/30 Write block 593 len 24576 data: 2 2 2 2 2 2 27/30 Write block 548 len 12288 data: 5 5 4 28/30 Write block 329 len 40960 data: 6 6 6 6 6 4 4 5 5 4 29/30 Read block 234 len 12288 30/30 Write block 505 len 40960 data: 5 4 6 5 5 5 5 4 4 4 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Read 9 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Read 10 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Read 6 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 8 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Write 3 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:06.719406308Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:06.719693148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1277,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 32 32 32 49 49 49 T T F <--- 3 060-079 31 31 31 48 48 48 F F F 4 080-099 33 33 33 50 50 50 F F F 5 100-119 34 34 34 52 52 52 F F F 6 120-139 36 36 34 60 60 57 F F F <--- 7 140-159 36 36 24 60 60 37 F F F <--- 8 160-179 28 28 28 42 42 42 F F F 9 180-199 34 34 34 55 55 55 F F F 10 200-219 27 27 27 40 40 40 F F F 11 220-239 36 36 31 62 62 48 T T F <--- 12 240-259 31 31 31 48 48 48 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 28 28 28 43 43 43 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 28 28 28 43 43 43 T T F <--- 17 340-359 23 23 23 35 35 35 F F F 18 360-379 34 34 34 55 55 55 F F F 19 380-399 33 33 33 50 50 50 T T F <--- 20 400-419 33 33 33 50 50 50 T T F <--- 21 420-439 36 36 34 61 61 56 F F F <--- 22 440-459 34 34 34 55 55 55 F F F 23 460-479 34 34 34 53 53 53 T T F <--- 24 480-499 36 36 35 59 59 58 F F F <--- 25 500-519 36 36 32 61 61 49 T T F <--- 26 520-539 34 34 34 55 55 55 F F F 27 540-559 34 34 34 55 55 55 T T F <--- 28 560-579 35 35 35 58 58 58 F F F 29 580-599 29 29 29 44 44 44 T T F <--- Max gen: 36, Max flush: 62 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 37 37 37 63 63 63 F F F 3 060-079 31 31 31 48 48 48 F F F 4 080-099 33 33 33 50 50 50 F F F 5 100-119 34 34 34 52 52 52 F F F 6 120-139 36 36 36 60 60 60 F F F 7 140-159 36 36 36 60 60 60 F F F 8 160-179 28 28 28 42 42 42 F F F 9 180-199 34 34 34 55 55 55 F F F 10 200-219 27 27 27 40 40 40 F F F 11 220-239 37 37 37 63 63 63 F F F 12 240-259 31 31 31 48 48 48 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 28 28 28 43 43 43 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 37 37 37 63 63 63 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 34 34 34 55 55 55 F F F 19 380-399 37 37 37 63 63 63 F F F 20 400-419 37 37 37 63 63 63 F F F 21 420-439 36 36 36 61 61 61 F F F 22 440-459 34 34 34 55 55 55 F F F 23 460-479 37 37 37 63 63 63 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 37 37 37 63 63 63 F F F 26 520-539 34 34 34 55 55 55 F F F 27 540-559 37 37 37 63 63 63 F F F 28 560-579 35 35 35 58 58 58 F F F 29 580-599 37 37 37 63 63 63 F F F Max gen: 37, Max flush: 63 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.963012087Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:10.970138162Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.970205948Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.977171114Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.977419297Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.977443685Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.977452645Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.977464444Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285} {"msg":"Crucible 8f93407d-4b9e-4664-9486-072dfc967a94 has session id: b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.977825139Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"Upstairs opts: Upstairs UUID: 8f93407d-4b9e-4664-9486-072dfc967a94, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.977841518Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"Crucible stats registered with UUID: 8f93407d-4b9e-4664-9486-072dfc967a94","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.977854127Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:10.977866836Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978027626Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978044084Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978054414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978202014Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978220002Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285} {"msg":"8f93407d-4b9e-4664-9486-072dfc967a94 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.9782532Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978263479Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"client":"0","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978273699Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"client":"1","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978283688Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"client":"2","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978293677Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978304107Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978355123Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"ds_connection connected from Ok(127.0.0.1:55173)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978380751Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"io task","client":"2","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"ds_connection connected from Ok(127.0.0.1:35300)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978392481Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"io task","client":"1","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"ds_connection connected from Ok(127.0.0.1:36560)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978487444Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"io task","client":"0","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.978973401Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"client":"1","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.97898703Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"client":"2","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979160548Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"client":"0","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979302819Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979328937Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"[0]R flush_numbers[0..12]: [30, 58, 63, 48, 50, 52, 60, 60, 42, 55, 40, 63]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979449229Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"downstairs","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"[0]R generation[0..12]: [20, 35, 37, 31, 33, 34, 36, 36, 28, 34, 27, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979460728Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"downstairs","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979472127Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"downstairs","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"[1]R flush_numbers[0..12]: [30, 58, 63, 48, 50, 52, 60, 60, 42, 55, 40, 63]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979482756Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"downstairs","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"[1]R generation[0..12]: [20, 35, 37, 31, 33, 34, 36, 36, 28, 34, 27, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979493446Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"downstairs","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979504565Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"downstairs","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"[2]R flush_numbers[0..12]: [30, 58, 63, 48, 50, 52, 60, 60, 42, 55, 40, 63]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979515264Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"downstairs","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"[2]R generation[0..12]: [20, 35, 37, 31, 33, 34, 36, 36, 28, 34, 27, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979525924Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"downstairs","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979539693Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"downstairs","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979549532Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"downstairs","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979559441Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"downstairs","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979569031Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"downstairs","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.97957879Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"":"downstairs","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979588839Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979597839Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"client":"0","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979607868Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"client":"1","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979617897Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"client":"2","":"downstairs","session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979628087Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979637586Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"8f93407d-4b9e-4664-9486-072dfc967a94 is now active with session: b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979647825Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979657085Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"session_id":"b9e7ba8c-d0ee-4696-a1d1-1f4163d6b099"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979666414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285} {"msg":"Activated sub_volume 8f93407d-4b9e-4664-9486-072dfc967a94","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979674883Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.979683263Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.98002013Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"task":"crutest"} 01/30 Write block 156 len 8192 data: 2 2 02/30 Read block 155 len 24576 03/30 Read block 85 len 32768 04/30 Read block 289 len 40960 05/30 Write block 242 len 12288 data: 6 6 6 06/30 Read block 277 len 8192 07/30 Flush 08/30 Write block 430 len 16384 data: 5 7 7 7 09/30 Write block 198 len 32768 data: 3 3 3 3 3 3 5 4 10/30 Read block 108 len 32768 11/30 Write block 183 len 16384 data: 3 4 6 5 12/30 Read block 169 len 16384 13/30 Read block 120 len 4096 14/30 Write block 373 len 24576 data: 3 2 3 3 5 7 15/30 Read block 415 len 20480 16/30 Read block 275 len 4096 17/30 Write block 448 len 12288 data: 5 4 4 18/30 Read block 21 len 16384 19/30 Write block 92 len 28672 data: 5 6 5 5 5 6 6 20/30 Read block 182 len 32768 21/30 Write block 164 len 32768 data: 4 5 6 7 7 8 9 8 22/30 Read block 226 len 4096 23/30 Write block 99 len 28672 data: 6 4 3 3 3 4 3 24/30 Read block 246 len 24576 25/30 Read block 13 len 24576 26/30 Read block 99 len 40960 27/30 Read block 139 len 36864 28/30 Write block 411 len 24576 data: 3 3 4 5 7 7 29/30 Write block 554 len 28672 data: 5 5 4 3 3 4 3 30/30 Write block 559 len 12288 data: 5 4 4 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Write 3 Sent Done Done false 1005 Acked Read 2 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Write 8 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 4 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 3 3 32 Done 4 27 27 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:10.988214582Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:10.988502583Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1285,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 37 37 37 63 63 63 F F F 3 060-079 31 31 31 48 48 48 F F F 4 080-099 33 33 33 50 50 50 F T T <--- 5 100-119 34 34 34 52 52 52 F T T <--- 6 120-139 36 36 36 60 60 60 F F F 7 140-159 36 38 38 60 64 64 T F F <--- 8 160-179 28 28 28 42 42 42 F T T <--- 9 180-199 34 34 34 55 55 55 F T T <--- 10 200-219 27 27 27 40 40 40 F T T <--- 11 220-239 37 37 37 63 63 63 F F F 12 240-259 31 38 38 48 64 64 F F F <--- 13 260-279 25 25 25 38 38 38 F F F 14 280-299 28 28 28 43 43 43 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 37 37 37 63 63 63 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 34 34 34 55 55 55 F T T <--- 19 380-399 37 37 37 63 63 63 F F F 20 400-419 37 37 37 63 63 63 F T T <--- 21 420-439 36 36 36 61 61 61 F T T <--- 22 440-459 34 34 34 55 55 55 F T T <--- 23 460-479 37 37 37 63 63 63 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 37 37 37 63 63 63 F F F 26 520-539 34 34 34 55 55 55 F F F 27 540-559 37 37 37 63 63 63 F T T <--- 28 560-579 35 35 35 58 58 58 F T T <--- 29 580-599 37 37 37 63 63 63 F F F Max gen: 38, Max flush: 64 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 35 35 35 58 58 58 F F F 2 040-059 37 37 37 63 63 63 F F F 3 060-079 31 31 31 48 48 48 F F F 4 080-099 39 39 39 65 65 65 F F F 5 100-119 39 39 39 65 65 65 F F F 6 120-139 36 36 36 60 60 60 F F F 7 140-159 38 38 38 64 64 64 F F F 8 160-179 39 39 39 65 65 65 F F F 9 180-199 39 39 39 65 65 65 F F F 10 200-219 39 39 39 65 65 65 F F F 11 220-239 37 37 37 63 63 63 F F F 12 240-259 38 38 38 64 64 64 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 28 28 28 43 43 43 F F F 15 300-319 30 30 30 47 47 47 F F F 16 320-339 37 37 37 63 63 63 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 37 37 37 63 63 63 F F F 20 400-419 39 39 39 65 65 65 F F F 21 420-439 39 39 39 65 65 65 F F F 22 440-459 39 39 39 65 65 65 F F F 23 460-479 37 37 37 63 63 63 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 37 37 37 63 63 63 F F F 26 520-539 34 34 34 55 55 55 F F F 27 540-559 39 39 39 65 65 65 F F F 28 560-579 39 39 39 65 65 65 F F F 29 580-599 37 37 37 63 63 63 F F F Max gen: 39, Max flush: 65 Begin loop 20 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.243527975Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:15.250649851Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.250717196Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.257475096Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.257721319Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.257744718Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.257753667Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.257765616Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293} {"msg":"Crucible f28d3d81-8d84-442f-87f3-0ed5dd2cc472 has session id: 795cc191-acd6-44c3-8cb9-e5ad92a2358f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258109053Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"Upstairs opts: Upstairs UUID: f28d3d81-8d84-442f-87f3-0ed5dd2cc472, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258128802Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"Crucible stats registered with UUID: f28d3d81-8d84-442f-87f3-0ed5dd2cc472","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258139501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:15.25815223Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258326418Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258343047Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258355846Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258494277Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258506336Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293} {"msg":"f28d3d81-8d84-442f-87f3-0ed5dd2cc472 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258540574Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258550263Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"client":"0","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258560262Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"client":"1","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258570252Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"client":"2","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258580081Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.25859991Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57737)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258618268Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"io task","client":"2","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258628748Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33493)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258667005Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"io task","client":"1","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"ds_connection connected from Ok(127.0.0.1:39189)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.258814105Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"io task","client":"0","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259269184Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"client":"1","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259298812Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"client":"2","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259365708Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"client":"0","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259576193Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259599642Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"[0]R flush_numbers[0..12]: [30, 58, 63, 48, 65, 65, 60, 64, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259651058Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"downstairs","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"[0]R generation[0..12]: [20, 35, 37, 31, 39, 39, 36, 38, 39, 39, 39, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259663047Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"downstairs","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259674777Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"downstairs","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"[1]R flush_numbers[0..12]: [30, 58, 63, 48, 65, 65, 60, 64, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259685896Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"downstairs","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"[1]R generation[0..12]: [20, 35, 37, 31, 39, 39, 36, 38, 39, 39, 39, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259696685Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"downstairs","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259707954Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"downstairs","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"[2]R flush_numbers[0..12]: [30, 58, 63, 48, 65, 65, 60, 64, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259718764Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"downstairs","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"[2]R generation[0..12]: [20, 35, 37, 31, 39, 39, 36, 38, 39, 39, 39, 37]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259729503Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"downstairs","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259740782Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"downstairs","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259750571Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"downstairs","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259760791Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"downstairs","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.25977047Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"downstairs","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259782839Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"":"downstairs","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259793308Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259802548Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"client":"0","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259812927Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"client":"1","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259823256Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"client":"2","":"downstairs","session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259833676Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259843205Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"f28d3d81-8d84-442f-87f3-0ed5dd2cc472 is now active with session: 795cc191-acd6-44c3-8cb9-e5ad92a2358f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259853564Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259863014Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259871073Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"session_id":"795cc191-acd6-44c3-8cb9-e5ad92a2358f"} {"msg":"Activated sub_volume f28d3d81-8d84-442f-87f3-0ed5dd2cc472","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259880553Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.259888992Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.26020718Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"task":"crutest"} 01/30 Write block 518 len 20480 data: 2 3 3 3 3 02/30 Write block 192 len 16384 data: 4 3 3 3 03/30 Write block 577 len 36864 data: 6 6 5 5 5 4 3 3 4 04/30 Write block 405 len 16384 data: 6 5 5 5 05/30 Read block 200 len 16384 06/30 Write block 337 len 12288 data: 6 5 4 07/30 Read block 574 len 28672 08/30 Write block 575 len 24576 data: 8 7 7 7 6 6 09/30 Read block 19 len 32768 10/30 Write block 281 len 12288 data: 3 3 3 11/30 Write block 153 len 32768 data: 2 2 2 3 3 3 5 4 12/30 Write block 126 len 4096 data: 5 13/30 Write block 555 len 24576 data: 6 5 4 4 6 5 14/30 Read block 340 len 36864 15/30 Write block 316 len 36864 data: 5 5 5 5 4 4 5 4 4 16/30 Write block 123 len 40960 data: 3 4 5 6 4 4 3 3 3 3 17/30 Write block 581 len 12288 data: 6 5 4 18/30 Read block 490 len 28672 19/30 Read block 372 len 12288 20/30 Write block 228 len 4096 data: 5 21/30 Flush 22/30 Write block 218 len 16384 data: 4 4 4 5 23/30 Write block 297 len 4096 data: 5 24/30 Write block 128 len 20480 data: 5 4 4 4 4 25/30 Flush 26/30 Write block 565 len 24576 data: 4 4 5 6 7 7 27/30 Write block 143 len 28672 data: 5 4 3 4 5 4 4 28/30 Write block 33 len 24576 data: 5 4 4 5 4 5 29/30 Write block 54 len 24576 data: 4 5 6 7 6 6 30/30 Write block 304 len 24576 data: 4 4 4 4 3 2 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Sent Done Done false 1001 Acked Write 4 Sent Done Done false 1002 Acked Write 9 Sent Done Done false 1003 Acked Write 4 Sent Done Done false 1004 Acked Read 4 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Read 7 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Write 8 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 7 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 3 3 36 Done 0 27 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:15.267595678Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:15.267892877Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1293,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 35 35 35 58 58 58 F T T <--- 2 040-059 37 37 37 63 63 63 F T T <--- 3 060-079 31 31 31 48 48 48 F F F 4 080-099 39 39 39 65 65 65 F F F 5 100-119 39 39 39 65 65 65 F F F 6 120-139 36 40 40 60 67 67 F F F <--- 7 140-159 38 40 40 64 66 66 F T T <--- 8 160-179 39 40 40 65 66 66 F F F <--- 9 180-199 39 40 40 65 66 66 F F F <--- 10 200-219 39 40 40 65 67 67 F F F <--- 11 220-239 37 40 40 63 67 67 F F F <--- 12 240-259 38 38 38 64 64 64 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 28 40 40 43 67 67 F F F <--- 15 300-319 30 40 40 47 66 66 F T T <--- 16 320-339 37 40 40 63 66 66 F F F <--- 17 340-359 23 23 23 35 35 35 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 37 37 37 63 63 63 F F F 20 400-419 39 40 40 65 66 66 F F F <--- 21 420-439 39 39 39 65 65 65 F F F 22 440-459 39 39 39 65 65 65 F F F 23 460-479 37 37 37 63 63 63 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 37 40 40 63 66 66 F F F <--- 26 520-539 34 40 40 55 66 66 F F F <--- 27 540-559 39 40 40 65 66 66 F F F <--- 28 560-579 39 40 40 65 66 66 F T T <--- 29 580-599 37 40 40 63 66 66 F F F <--- Max gen: 40, Max flush: 67 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 20 20 30 30 30 F F F 1 020-039 41 41 41 68 68 68 F F F 2 040-059 41 41 41 68 68 68 F F F 3 060-079 31 31 31 48 48 48 F F F 4 080-099 39 39 39 65 65 65 F F F 5 100-119 39 39 39 65 65 65 F F F 6 120-139 40 40 40 67 67 67 F F F 7 140-159 41 41 41 68 68 68 F F F 8 160-179 40 40 40 66 66 66 F F F 9 180-199 40 40 40 66 66 66 F F F 10 200-219 40 40 40 67 67 67 F F F 11 220-239 40 40 40 67 67 67 F F F 12 240-259 38 38 38 64 64 64 F F F 13 260-279 25 25 25 38 38 38 F F F 14 280-299 40 40 40 67 67 67 F F F 15 300-319 41 41 41 68 68 68 F F F 16 320-339 40 40 40 66 66 66 F F F 17 340-359 23 23 23 35 35 35 F F F 18 360-379 39 39 39 65 65 65 F F F 19 380-399 37 37 37 63 63 63 F F F 20 400-419 40 40 40 66 66 66 F F F 21 420-439 39 39 39 65 65 65 F F F 22 440-459 39 39 39 65 65 65 F F F 23 460-479 37 37 37 63 63 63 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 40 40 40 66 66 66 F F F 26 520-539 40 40 40 66 66 66 F F F 27 540-559 40 40 40 66 66 66 F F F 28 560-579 41 41 41 68 68 68 F F F 29 580-599 40 40 40 66 66 66 F F F Max gen: 41, Max flush: 68 Begin loop 21 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.524644362Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:19.531765798Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.531828254Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.538600223Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.538856015Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.538879794Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.538888533Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.538900302Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301} {"msg":"Crucible 0b6e7919-c7bd-4579-b36b-6dbe79714f4d has session id: 185f4eb7-96e4-428c-bb8c-7dac2697b433","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539281596Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"Upstairs opts: Upstairs UUID: 0b6e7919-c7bd-4579-b36b-6dbe79714f4d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539320794Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"Crucible stats registered with UUID: 0b6e7919-c7bd-4579-b36b-6dbe79714f4d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539332273Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:19.539342042Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539506951Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539538629Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539549618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.53967729Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539702168Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301} {"msg":"0b6e7919-c7bd-4579-b36b-6dbe79714f4d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539718777Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539730346Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"client":"0","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539740485Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"client":"1","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539750405Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"client":"2","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539760374Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539770823Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539781152Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"ds_connection connected from Ok(127.0.0.1:53311)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539951881Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"io task","client":"2","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"ds_connection connected from Ok(127.0.0.1:57113)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.53996702Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"io task","client":"0","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"ds_connection connected from Ok(127.0.0.1:55058)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.539977879Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"io task","client":"1","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.540580958Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"client":"2","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.540601087Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"client":"1","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.540643514Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"client":"0","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.540880588Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.540914625Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"[0]R flush_numbers[0..12]: [30, 68, 68, 48, 65, 65, 67, 68, 66, 66, 67, 67]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.540967952Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"downstairs","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"[0]R generation[0..12]: [20, 41, 41, 31, 39, 39, 40, 41, 40, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.540981931Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"downstairs","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.54099328Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"downstairs","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"[1]R flush_numbers[0..12]: [30, 68, 68, 48, 65, 65, 67, 68, 66, 66, 67, 67]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541013959Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"downstairs","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"[1]R generation[0..12]: [20, 41, 41, 31, 39, 39, 40, 41, 40, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541024828Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"downstairs","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541035977Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"downstairs","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"[2]R flush_numbers[0..12]: [30, 68, 68, 48, 65, 65, 67, 68, 66, 66, 67, 67]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541046646Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"downstairs","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"[2]R generation[0..12]: [20, 41, 41, 31, 39, 39, 40, 41, 40, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541057186Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"downstairs","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541068415Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"downstairs","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541078124Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"downstairs","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541088124Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"downstairs","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541098073Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"downstairs","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541107882Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"":"downstairs","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541118261Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541127521Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"client":"0","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.54113783Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"client":"1","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541149159Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"client":"2","":"downstairs","session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541159459Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541168848Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"0b6e7919-c7bd-4579-b36b-6dbe79714f4d is now active with session: 185f4eb7-96e4-428c-bb8c-7dac2697b433","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541179067Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541188497Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541196526Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"session_id":"185f4eb7-96e4-428c-bb8c-7dac2697b433"} {"msg":"Activated sub_volume 0b6e7919-c7bd-4579-b36b-6dbe79714f4d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541205975Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541214345Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.541542723Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"task":"crutest"} 01/30 Write block 232 len 24576 data: 6 5 4 5 5 3 02/30 Read block 494 len 20480 03/30 Write block 282 len 16384 data: 4 4 4 3 04/30 Read block 467 len 24576 05/30 Read block 486 len 32768 06/30 Read block 380 len 12288 07/30 Write block 311 len 40960 data: 3 4 4 4 4 6 6 6 6 5 08/30 Read block 75 len 28672 09/30 Read block 48 len 36864 10/30 Read block 7 len 8192 11/30 Flush 12/30 Write block 582 len 28672 data: 6 5 4 5 4 5 3 13/30 Read block 453 len 40960 14/30 Read block 316 len 24576 15/30 Read block 213 len 12288 16/30 Write block 303 len 20480 data: 4 5 5 5 5 17/30 Write block 200 len 36864 data: 4 4 4 4 6 5 4 4 6 18/30 Write block 319 len 12288 data: 7 6 5 19/30 Write block 306 len 16384 data: 6 6 4 3 20/30 Read block 84 len 36864 21/30 Write block 275 len 20480 data: 2 3 2 2 2 22/30 Write block 371 len 24576 data: 3 4 4 3 4 4 23/30 Write block 5 len 24576 data: 3 3 4 4 4 4 24/30 Write block 39 len 12288 data: 7 8 8 25/30 Flush 26/30 Flush 27/30 Write block 173 len 32768 data: 7 7 5 4 2 2 2 2 28/30 Read block 122 len 12288 29/30 Read block 312 len 36864 30/30 Write block 341 len 24576 data: 3 2 2 3 3 3 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 5 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 6 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 7 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Read 9 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 1 29 Done 3 29 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:19.550673531Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:19.550962532Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1301,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 20 42 42 30 70 70 F F F <--- 1 020-039 41 42 42 68 70 70 F F F <--- 2 040-059 41 42 42 68 70 70 F F F <--- 3 060-079 31 31 31 48 48 48 F F F 4 080-099 39 39 39 65 65 65 F F F 5 100-119 39 39 39 65 65 65 F F F 6 120-139 40 40 40 67 67 67 F F F 7 140-159 41 41 41 68 68 68 F F F 8 160-179 40 40 40 66 66 66 F T T <--- 9 180-199 40 40 40 66 66 66 F T T <--- 10 200-219 40 42 42 67 70 70 F F F <--- 11 220-239 40 42 42 67 69 69 T F F <--- 12 240-259 38 38 38 64 64 64 F F F 13 260-279 25 42 42 38 70 70 F F F <--- 14 280-299 40 42 42 67 69 69 T F F <--- 15 300-319 41 42 42 68 70 70 F F F <--- 16 320-339 40 42 42 66 70 70 F F F <--- 17 340-359 23 23 23 35 35 35 F T T <--- 18 360-379 39 42 42 65 70 70 F F F <--- 19 380-399 37 37 37 63 63 63 F F F 20 400-419 40 40 40 66 66 66 F F F 21 420-439 39 39 39 65 65 65 F F F 22 440-459 39 39 39 65 65 65 F F F 23 460-479 37 37 37 63 63 63 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 40 40 40 66 66 66 F F F 26 520-539 40 40 40 66 66 66 F F F 27 540-559 40 40 40 66 66 66 F F F 28 560-579 41 41 41 68 68 68 F F F 29 580-599 40 42 42 66 70 70 F F F <--- Max gen: 42, Max flush: 70 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 70 70 70 F F F 1 020-039 42 42 42 70 70 70 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 31 31 31 48 48 48 F F F 4 080-099 39 39 39 65 65 65 F F F 5 100-119 39 39 39 65 65 65 F F F 6 120-139 40 40 40 67 67 67 F F F 7 140-159 41 41 41 68 68 68 F F F 8 160-179 43 43 43 71 71 71 F F F 9 180-199 43 43 43 71 71 71 F F F 10 200-219 42 42 42 70 70 70 F F F 11 220-239 42 42 42 69 69 69 F F F 12 240-259 38 38 38 64 64 64 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 42 42 42 69 69 69 F F F 15 300-319 42 42 42 70 70 70 F F F 16 320-339 42 42 42 70 70 70 F F F 17 340-359 43 43 43 71 71 71 F F F 18 360-379 42 42 42 70 70 70 F F F 19 380-399 37 37 37 63 63 63 F F F 20 400-419 40 40 40 66 66 66 F F F 21 420-439 39 39 39 65 65 65 F F F 22 440-459 39 39 39 65 65 65 F F F 23 460-479 37 37 37 63 63 63 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 40 40 40 66 66 66 F F F 26 520-539 40 40 40 66 66 66 F F F 27 540-559 40 40 40 66 66 66 F F F 28 560-579 41 41 41 68 68 68 F F F 29 580-599 42 42 42 70 70 70 F F F Max gen: 43, Max flush: 71 Begin loop 22 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.804755738Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:23.811915481Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.811982176Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.818894166Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.819159708Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.819183926Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.819192746Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.819206745Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309} {"msg":"Crucible a13b0025-47ad-4ea1-bc9c-971d93e3d81c has session id: 6d1dd17c-862b-475a-b0f8-dce086d99bf1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.819594158Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"Upstairs opts: Upstairs UUID: a13b0025-47ad-4ea1-bc9c-971d93e3d81c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.819614057Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"Crucible stats registered with UUID: a13b0025-47ad-4ea1-bc9c-971d93e3d81c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.819624666Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:23.819637415Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.819802284Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.819819203Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.819829802Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.819971383Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.819985542Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309} {"msg":"a13b0025-47ad-4ea1-bc9c-971d93e3d81c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.820029959Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.820039618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"client":"0","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.820049677Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"client":"1","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.820059617Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"client":"2","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.820069566Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.820083535Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.820133762Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:35813)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.820145031Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"io task","client":"2","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:58841)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.82015577Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"io task","client":"1","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"ds_connection connected from Ok(127.0.0.1:64538)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.820254053Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"io task","client":"0","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.820809026Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"client":"2","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.820857072Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"client":"1","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821017411Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"client":"0","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821161042Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821171441Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"[0]R flush_numbers[0..12]: [70, 70, 70, 48, 65, 65, 67, 68, 71, 71, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821297782Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"downstairs","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"[0]R generation[0..12]: [42, 42, 42, 31, 39, 39, 40, 41, 43, 43, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821309342Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"downstairs","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821320921Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"downstairs","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"[1]R flush_numbers[0..12]: [70, 70, 70, 48, 65, 65, 67, 68, 71, 71, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.82133185Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"downstairs","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"[1]R generation[0..12]: [42, 42, 42, 31, 39, 39, 40, 41, 43, 43, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821342889Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"downstairs","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821356798Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"downstairs","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"[2]R flush_numbers[0..12]: [70, 70, 70, 48, 65, 65, 67, 68, 71, 71, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821367528Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"downstairs","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"[2]R generation[0..12]: [42, 42, 42, 31, 39, 39, 40, 41, 43, 43, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821378177Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"downstairs","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821389236Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"downstairs","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821398895Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"downstairs","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821408965Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"downstairs","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821418594Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"downstairs","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821428193Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"":"downstairs","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821438253Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821447422Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"client":"0","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821457272Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"client":"1","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821467361Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"client":"2","":"downstairs","session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.8214775Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821487399Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"a13b0025-47ad-4ea1-bc9c-971d93e3d81c is now active with session: 6d1dd17c-862b-475a-b0f8-dce086d99bf1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821497659Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821508378Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821516287Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"session_id":"6d1dd17c-862b-475a-b0f8-dce086d99bf1"} {"msg":"Activated sub_volume a13b0025-47ad-4ea1-bc9c-971d93e3d81c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821525767Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821534316Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.821852285Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"task":"crutest"} 01/30 Write block 167 len 36864 data: 8 8 9 10 9 7 8 8 6 02/30 Read block 150 len 28672 03/30 Flush 04/30 Write block 375 len 8192 data: 5 5 05/30 Write block 419 len 40960 data: 5 4 4 4 4 3 5 5 6 5 06/30 Read block 390 len 32768 07/30 Read block 137 len 12288 08/30 Write block 390 len 8192 data: 2 3 09/30 Read block 121 len 8192 10/30 Read block 489 len 16384 11/30 Write block 441 len 12288 data: 4 4 3 12/30 Read block 107 len 28672 13/30 Read block 536 len 4096 14/30 Write block 522 len 16384 data: 4 3 3 4 15/30 Read block 267 len 40960 16/30 Write block 30 len 28672 data: 4 5 3 6 5 5 6 17/30 Read block 16 len 24576 18/30 Read block 215 len 40960 19/30 Read block 536 len 20480 20/30 Read block 144 len 36864 21/30 Read block 569 len 32768 22/30 Write block 18 len 4096 data: 5 23/30 Read block 18 len 20480 24/30 Read block 300 len 24576 25/30 Write block 317 len 32768 data: 7 7 8 7 6 6 5 5 26/30 Write block 125 len 4096 data: 6 27/30 Read block 579 len 8192 28/30 Read block 142 len 36864 29/30 Read block 160 len 32768 30/30 Write block 465 len 32768 data: 8 8 5 5 5 5 4 3 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Sent Done Done false 1001 Acked Read 7 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Write 2 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1002)) Some(JobId(1002)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:23.830569991Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:23.830864311Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1309,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 70 70 70 F T T <--- 1 020-039 42 42 42 70 70 70 F T T <--- 2 040-059 42 42 42 70 70 70 F F F 3 060-079 31 31 31 48 48 48 F F F 4 080-099 39 39 39 65 65 65 F F F 5 100-119 39 39 39 65 65 65 F F F 6 120-139 40 40 40 67 67 67 F T T <--- 7 140-159 41 41 41 68 68 68 F F F 8 160-179 43 44 44 71 72 72 F F F <--- 9 180-199 43 43 43 71 71 71 F F F 10 200-219 42 42 42 70 70 70 F F F 11 220-239 42 42 42 69 69 69 F F F 12 240-259 38 38 38 64 64 64 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 42 42 42 69 69 69 F F F 15 300-319 42 42 42 70 70 70 F T T <--- 16 320-339 42 42 42 70 70 70 F T T <--- 17 340-359 43 43 43 71 71 71 F F F 18 360-379 42 42 42 70 70 70 F T T <--- 19 380-399 37 37 37 63 63 63 F T T <--- 20 400-419 40 40 40 66 66 66 F T T <--- 21 420-439 39 39 39 65 65 65 F T T <--- 22 440-459 39 39 39 65 65 65 F T T <--- 23 460-479 37 37 37 63 63 63 F T T <--- 24 480-499 36 36 36 59 59 59 F F F 25 500-519 40 40 40 66 66 66 F F F 26 520-539 40 40 40 66 66 66 F T T <--- 27 540-559 40 40 40 66 66 66 F F F 28 560-579 41 41 41 68 68 68 F F F 29 580-599 42 42 42 70 70 70 F F F Max gen: 44, Max flush: 72 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 45 45 73 73 73 F F F 1 020-039 45 45 45 73 73 73 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 31 31 31 48 48 48 F F F 4 080-099 39 39 39 65 65 65 F F F 5 100-119 39 39 39 65 65 65 F F F 6 120-139 45 45 45 73 73 73 F F F 7 140-159 41 41 41 68 68 68 F F F 8 160-179 44 44 44 72 72 72 F F F 9 180-199 43 43 43 71 71 71 F F F 10 200-219 42 42 42 70 70 70 F F F 11 220-239 42 42 42 69 69 69 F F F 12 240-259 38 38 38 64 64 64 F F F 13 260-279 42 42 42 70 70 70 F F F 14 280-299 42 42 42 69 69 69 F F F 15 300-319 45 45 45 73 73 73 F F F 16 320-339 45 45 45 73 73 73 F F F 17 340-359 43 43 43 71 71 71 F F F 18 360-379 45 45 45 73 73 73 F F F 19 380-399 45 45 45 73 73 73 F F F 20 400-419 45 45 45 73 73 73 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 45 45 45 73 73 73 F F F 23 460-479 45 45 45 73 73 73 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 40 40 40 66 66 66 F F F 26 520-539 45 45 45 73 73 73 F F F 27 540-559 40 40 40 66 66 66 F F F 28 560-579 41 41 41 68 68 68 F F F 29 580-599 42 42 42 70 70 70 F F F Max gen: 45, Max flush: 73 Begin loop 23 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.084583013Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:28.091720057Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.091778083Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.098474587Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.098714101Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.098739989Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.098748929Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.098760908Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317} {"msg":"Crucible 9efabb7f-1cc0-4305-8b92-8db848db274b has session id: 12a99470-464f-4cd9-bd8b-756d9436a74e","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099083676Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"Upstairs opts: Upstairs UUID: 9efabb7f-1cc0-4305-8b92-8db848db274b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099113474Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"Crucible stats registered with UUID: 9efabb7f-1cc0-4305-8b92-8db848db274b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099125193Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:28.099137862Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099305811Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.09932403Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099334869Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099436892Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099455021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317} {"msg":"9efabb7f-1cc0-4305-8b92-8db848db274b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099492618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099502677Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"client":"0","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099512847Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"client":"1","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099522966Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"client":"2","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099532995Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099543575Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49546)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099581602Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"io task","client":"2","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099592581Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36825)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099603421Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"io task","client":"1","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"ds_connection connected from Ok(127.0.0.1:45672)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.099667046Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"io task","client":"0","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100186991Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"client":"2","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100211889Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"client":"1","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100319062Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"client":"0","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100508699Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100534367Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"[0]R flush_numbers[0..12]: [73, 73, 70, 48, 65, 65, 73, 68, 72, 71, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100583374Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"downstairs","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"[0]R generation[0..12]: [45, 45, 42, 31, 39, 39, 45, 41, 44, 43, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100595033Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"downstairs","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100606502Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"downstairs","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"[1]R flush_numbers[0..12]: [73, 73, 70, 48, 65, 65, 73, 68, 72, 71, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100617402Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"downstairs","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"[1]R generation[0..12]: [45, 45, 42, 31, 39, 39, 45, 41, 44, 43, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100628121Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"downstairs","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.10063935Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"downstairs","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"[2]R flush_numbers[0..12]: [73, 73, 70, 48, 65, 65, 73, 68, 72, 71, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100650469Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"downstairs","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"[2]R generation[0..12]: [45, 45, 42, 31, 39, 39, 45, 41, 44, 43, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100661329Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"downstairs","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100672428Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"downstairs","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100684297Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"downstairs","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100694436Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"downstairs","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100704076Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"downstairs","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100713755Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"":"downstairs","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100723814Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100732874Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"client":"0","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100742773Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"client":"1","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100752692Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"client":"2","":"downstairs","session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100762952Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100772421Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"9efabb7f-1cc0-4305-8b92-8db848db274b is now active with session: 12a99470-464f-4cd9-bd8b-756d9436a74e","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.10078269Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.1007922Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100800569Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"session_id":"12a99470-464f-4cd9-bd8b-756d9436a74e"} {"msg":"Activated sub_volume 9efabb7f-1cc0-4305-8b92-8db848db274b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.100810339Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-07-28T04:40:28.100818878Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.10107727Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"task":"crutest"} 01/30 Write block 188 len 32768 data: 6 6 5 5 5 4 4 4 02/30 Read block 145 len 36864 03/30 Write block 528 len 24576 data: 3 4 4 4 5 5 04/30 Read block 563 len 12288 05/30 Flush 06/30 Write block 329 len 32768 data: 7 7 7 7 7 5 5 6 07/30 Write block 268 len 24576 data: 3 3 2 2 2 2 08/30 Read block 292 len 40960 09/30 Read block 402 len 36864 10/30 Read block 279 len 16384 11/30 Read block 352 len 4096 12/30 Read block 148 len 8192 13/30 Read block 242 len 28672 14/30 Read block 407 len 24576 15/30 Read block 564 len 32768 16/30 Flush 17/30 Write block 66 len 8192 data: 3 3 18/30 Write block 78 len 36864 data: 6 5 6 4 3 2 2 2 3 19/30 Read block 192 len 20480 20/30 Write block 1 len 28672 data: 2 2 2 3 4 4 5 21/30 Write block 356 len 32768 data: 4 5 6 7 6 7 5 5 22/30 Read block 219 len 8192 23/30 Flush 24/30 Write block 259 len 4096 data: 2 25/30 Read block 158 len 16384 26/30 Read block 456 len 24576 27/30 Read block 404 len 16384 28/30 Write block 162 len 24576 data: 4 4 5 6 7 9 29/30 Write block 555 len 28672 data: 7 6 5 5 7 6 5 30/30 Read block 215 len 40960 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 9 Sent Done Done false 1002 Acked Write 6 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 10 Sent Done Done false 1008 Acked Read 9 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 6 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Read 4 Sent Done Done false 1025 Acked Read 6 Sent Done Done false 1026 Acked Read 4 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 1 30 Done 1 30 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:28.110304673Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:28.110614131Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1317,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 45 46 46 73 76 76 F F F <--- 1 020-039 45 45 45 73 73 73 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 31 46 46 48 76 76 F F F <--- 4 080-099 39 46 46 65 76 76 F F F <--- 5 100-119 39 39 39 65 65 65 F F F 6 120-139 45 45 45 73 73 73 F F F 7 140-159 41 41 41 68 68 68 F F F 8 160-179 44 44 44 72 72 72 F T T <--- 9 180-199 43 46 46 71 74 74 T F F <--- 10 200-219 42 42 42 70 70 70 F F F 11 220-239 42 42 42 69 69 69 F F F 12 240-259 38 38 38 64 64 64 F T T <--- 13 260-279 42 46 46 70 75 75 F F F <--- 14 280-299 42 42 42 69 69 69 F F F 15 300-319 45 45 45 73 73 73 F F F 16 320-339 45 46 46 73 75 75 F F F <--- 17 340-359 43 46 46 71 76 76 F F F <--- 18 360-379 45 46 46 73 76 76 F F F <--- 19 380-399 45 45 45 73 73 73 F F F 20 400-419 45 45 45 73 73 73 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 45 45 45 73 73 73 F F F 23 460-479 45 45 45 73 73 73 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 40 40 40 66 66 66 F F F 26 520-539 45 46 46 73 74 74 F F F <--- 27 540-559 40 40 40 66 66 66 F T T <--- 28 560-579 41 41 41 68 68 68 F T T <--- 29 580-599 42 42 42 70 70 70 F F F Max gen: 46, Max flush: 76 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 76 76 76 F F F 1 020-039 45 45 45 73 73 73 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 46 46 46 76 76 76 F F F 4 080-099 46 46 46 76 76 76 F F F 5 100-119 39 39 39 65 65 65 F F F 6 120-139 45 45 45 73 73 73 F F F 7 140-159 41 41 41 68 68 68 F F F 8 160-179 47 47 47 77 77 77 F F F 9 180-199 46 46 46 74 74 74 F F F 10 200-219 42 42 42 70 70 70 F F F 11 220-239 42 42 42 69 69 69 F F F 12 240-259 47 47 47 77 77 77 F F F 13 260-279 46 46 46 75 75 75 F F F 14 280-299 42 42 42 69 69 69 F F F 15 300-319 45 45 45 73 73 73 F F F 16 320-339 46 46 46 75 75 75 F F F 17 340-359 46 46 46 76 76 76 F F F 18 360-379 46 46 46 76 76 76 F F F 19 380-399 45 45 45 73 73 73 F F F 20 400-419 45 45 45 73 73 73 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 45 45 45 73 73 73 F F F 23 460-479 45 45 45 73 73 73 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 40 40 40 66 66 66 F F F 26 520-539 46 46 46 74 74 74 F F F 27 540-559 47 47 47 77 77 77 F F F 28 560-579 47 47 47 77 77 77 F F F 29 580-599 42 42 42 70 70 70 F F F Max gen: 47, Max flush: 77 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.357917629Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.366251912Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.366531883Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.366558801Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.366567701Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.36657963Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325} {"msg":"Crucible 3a0d4b55-366a-4a27-855b-ba7918795d04 has session id: 9a1ec35c-7c88-477a-88a9-1e7478385a0e","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.366963564Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"Upstairs opts: Upstairs UUID: 3a0d4b55-366a-4a27-855b-ba7918795d04, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.366993882Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"Crucible stats registered with UUID: 3a0d4b55-366a-4a27-855b-ba7918795d04","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.36701614Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:32.367026099Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.367185329Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.367215387Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.367226806Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.367343208Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.367374246Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325} {"msg":"3a0d4b55-366a-4a27-855b-ba7918795d04 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.367398484Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.367410403Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"client":"0","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.367420593Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"client":"1","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.367430762Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"client":"2","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.367440701Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.367451101Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.36746139Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61007)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.367512696Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"io task","client":"0","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35995)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.367539095Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"io task","client":"1","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41203)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.36760168Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"io task","client":"2","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.36819946Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"client":"0","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368304073Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"client":"2","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368330451Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"client":"1","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368506179Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368579704Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"[0]R flush_numbers[0..12]: [76, 73, 70, 76, 76, 65, 73, 68, 77, 74, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.36863508Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"downstairs","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"[0]R generation[0..12]: [46, 45, 42, 46, 46, 39, 45, 41, 47, 46, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368649109Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"downstairs","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368660548Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"downstairs","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"[1]R flush_numbers[0..12]: [76, 73, 70, 76, 76, 65, 73, 68, 77, 74, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368671378Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"downstairs","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"[1]R generation[0..12]: [46, 45, 42, 46, 46, 39, 45, 41, 47, 46, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368682137Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"downstairs","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368693196Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"downstairs","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"[2]R flush_numbers[0..12]: [76, 73, 70, 76, 76, 65, 73, 68, 77, 74, 70, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368703815Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"downstairs","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"[2]R generation[0..12]: [46, 45, 42, 46, 46, 39, 45, 41, 47, 46, 42, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368714345Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"downstairs","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368725474Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"downstairs","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368735303Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"downstairs","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368745293Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"downstairs","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368754942Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"downstairs","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368764731Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"":"downstairs","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.36877492Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368784Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"client":"0","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368794389Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"client":"1","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368805548Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"client":"2","":"downstairs","session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368816088Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368825637Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"3a0d4b55-366a-4a27-855b-ba7918795d04 is now active with session: 9a1ec35c-7c88-477a-88a9-1e7478385a0e","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368835826Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368845246Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368853125Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"session_id":"9a1ec35c-7c88-477a-88a9-1e7478385a0e"} {"msg":"Activated sub_volume 3a0d4b55-366a-4a27-855b-ba7918795d04","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368862505Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.368873954Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.369140326Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"task":"crutest"} 01/30 Write block 339 len 36864 data: 5 4 4 3 3 4 4 4 3 02/30 Read block 543 len 36864 03/30 Write block 591 len 4096 data: 4 04/30 Write block 287 len 4096 data: 4 05/30 Read block 43 len 4096 06/30 Read block 476 len 28672 07/30 Write block 366 len 8192 data: 6 4 08/30 Write block 535 len 8192 data: 4 3 09/30 Write block 292 len 8192 data: 2 2 10/30 Read block 449 len 20480 11/30 Write block 200 len 28672 data: 5 5 5 5 7 6 5 12/30 Write block 103 len 8192 data: 4 5 13/30 Read block 547 len 36864 14/30 Read block 373 len 4096 15/30 Write block 78 len 16384 data: 7 6 7 5 16/30 Read block 101 len 12288 17/30 Write block 257 len 24576 data: 2 2 3 2 3 3 18/30 Write block 141 len 16384 data: 6 6 6 5 19/30 Read block 456 len 20480 20/30 Read block 30 len 36864 21/30 Read block 339 len 28672 22/30 Write block 368 len 40960 data: 5 6 5 4 5 5 4 6 6 6 23/30 Read block 246 len 8192 24/30 Read block 208 len 36864 25/30 Read block 597 len 4096 26/30 Read block 380 len 28672 27/30 Read block 210 len 32768 28/30 Write block 569 len 4096 data: 8 29/30 Read block 532 len 28672 30/30 Write block 349 len 32768 data: 4 4 4 4 5 4 4 5 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Sent Done false 1001 Acked Read 9 Done Sent Done false 1002 Acked Write 1 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Write 2 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Read 1 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Read 3 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:32.376994391Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:32.377292731Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1325,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 76 76 76 F F F 1 020-039 45 45 45 73 73 73 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 46 46 46 76 76 76 T F T <--- 4 080-099 46 46 46 76 76 76 T F T <--- 5 100-119 39 39 39 65 65 65 T F T <--- 6 120-139 45 45 45 73 73 73 F F F 7 140-159 41 41 41 68 68 68 T F T <--- 8 160-179 47 47 47 77 77 77 F F F 9 180-199 46 46 46 74 74 74 F F F 10 200-219 42 42 42 70 70 70 T F T <--- 11 220-239 42 42 42 69 69 69 F F F 12 240-259 47 47 47 77 77 77 T F T <--- 13 260-279 46 46 46 75 75 75 T F T <--- 14 280-299 42 42 42 69 69 69 T F T <--- 15 300-319 45 45 45 73 73 73 F F F 16 320-339 46 46 46 75 75 75 T F T <--- 17 340-359 46 46 46 76 76 76 T F T <--- 18 360-379 46 46 46 76 76 76 T F T <--- 19 380-399 45 45 45 73 73 73 F F F 20 400-419 45 45 45 73 73 73 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 45 45 45 73 73 73 F F F 23 460-479 45 45 45 73 73 73 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 40 40 40 66 66 66 F F F 26 520-539 46 46 46 74 74 74 T F T <--- 27 540-559 47 47 47 77 77 77 F F F 28 560-579 47 47 47 77 77 77 T F T <--- 29 580-599 42 42 42 70 70 70 T F T <--- Max gen: 47, Max flush: 77 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 76 76 76 F F F 1 020-039 45 45 45 73 73 73 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 48 48 48 78 78 78 F F F 4 080-099 48 48 48 78 78 78 F F F 5 100-119 48 48 48 78 78 78 F F F 6 120-139 45 45 45 73 73 73 F F F 7 140-159 48 48 48 78 78 78 F F F 8 160-179 47 47 47 77 77 77 F F F 9 180-199 46 46 46 74 74 74 F F F 10 200-219 48 48 48 78 78 78 F F F 11 220-239 42 42 42 69 69 69 F F F 12 240-259 48 48 48 78 78 78 F F F 13 260-279 48 48 48 78 78 78 F F F 14 280-299 48 48 48 78 78 78 F F F 15 300-319 45 45 45 73 73 73 F F F 16 320-339 48 48 48 78 78 78 F F F 17 340-359 48 48 48 78 78 78 F F F 18 360-379 48 48 48 78 78 78 F F F 19 380-399 45 45 45 73 73 73 F F F 20 400-419 45 45 45 73 73 73 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 45 45 45 73 73 73 F F F 23 460-479 45 45 45 73 73 73 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 40 40 40 66 66 66 F F F 26 520-539 48 48 48 78 78 78 F F F 27 540-559 47 47 47 77 77 77 F F F 28 560-579 48 48 48 78 78 78 F F F 29 580-599 48 48 48 78 78 78 F F F Max gen: 48, Max flush: 78 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.625416983Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.63354477Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.633825201Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.633850929Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.633860108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.633872088Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333} {"msg":"Crucible e4ca8dca-d3f5-428e-af27-0469892762ea has session id: 4a5e33c2-3449-4fbf-8d92-6018f2086417","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634229853Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"Upstairs opts: Upstairs UUID: e4ca8dca-d3f5-428e-af27-0469892762ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634261641Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"Crucible stats registered with UUID: e4ca8dca-d3f5-428e-af27-0469892762ea","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.6342734Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:36.63428354Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634461897Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634492865Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634504445Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634589299Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634603328Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333} {"msg":"e4ca8dca-d3f5-428e-af27-0469892762ea active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634625546Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634634856Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"client":"0","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634645025Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"client":"1","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634654994Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"client":"2","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634664814Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634690512Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634701431Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"ds_connection connected from Ok(127.0.0.1:43753)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634755267Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"io task","client":"2","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"ds_connection connected from Ok(127.0.0.1:51780)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634783166Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"io task","client":"0","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"ds_connection connected from Ok(127.0.0.1:37408)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.634888828Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"io task","client":"1","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635359436Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"client":"0","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635419262Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"client":"2","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635445731Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"client":"1","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635758059Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635768229Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"[0]R flush_numbers[0..12]: [76, 73, 70, 78, 78, 78, 73, 78, 77, 74, 78, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635793387Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"downstairs","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"[0]R generation[0..12]: [46, 45, 42, 48, 48, 48, 45, 48, 47, 46, 48, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635804876Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"downstairs","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635816275Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"downstairs","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"[1]R flush_numbers[0..12]: [76, 73, 70, 78, 78, 78, 73, 78, 77, 74, 78, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635827015Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"downstairs","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"[1]R generation[0..12]: [46, 45, 42, 48, 48, 48, 45, 48, 47, 46, 48, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635837614Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"downstairs","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635848823Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"downstairs","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"[2]R flush_numbers[0..12]: [76, 73, 70, 78, 78, 78, 73, 78, 77, 74, 78, 69]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635859442Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"downstairs","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"[2]R generation[0..12]: [46, 45, 42, 48, 48, 48, 45, 48, 47, 46, 48, 42]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635872172Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"downstairs","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635883211Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"downstairs","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"Max found gen is 49","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.63589313Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"downstairs","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"Generation requested: 50 >= found:49","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635903349Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"downstairs","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635913059Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"downstairs","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635922688Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"":"downstairs","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635932627Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635941677Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"client":"0","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635951566Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"client":"1","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635961495Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"client":"2","":"downstairs","session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635971625Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635981114Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"e4ca8dca-d3f5-428e-af27-0469892762ea is now active with session: 4a5e33c2-3449-4fbf-8d92-6018f2086417","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.635991483Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.636000873Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.636008882Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"session_id":"4a5e33c2-3449-4fbf-8d92-6018f2086417"} {"msg":"Activated sub_volume e4ca8dca-d3f5-428e-af27-0469892762ea","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.636018372Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.636027961Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.636365108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"task":"crutest"} 01/30 Write block 366 len 40960 data: 7 5 6 7 6 5 6 6 5 7 02/30 Read block 213 len 36864 03/30 Flush 04/30 Write block 231 len 8192 data: 6 7 05/30 Write block 6 len 4096 data: 5 06/30 Write block 355 len 32768 data: 5 6 6 7 8 7 8 6 07/30 Read block 357 len 28672 08/30 Read block 129 len 32768 09/30 Read block 420 len 12288 10/30 Read block 319 len 24576 11/30 Read block 346 len 32768 12/30 Write block 504 len 24576 data: 5 6 5 7 6 6 13/30 Read block 580 len 4096 14/30 Write block 18 len 20480 data: 6 6 6 6 4 15/30 Flush 16/30 Write block 317 len 32768 data: 8 8 9 8 7 7 6 6 17/30 Read block 293 len 32768 18/30 Read block 187 len 12288 19/30 Write block 73 len 24576 data: 6 6 5 6 7 8 20/30 Write block 308 len 28672 data: 5 4 2 4 5 5 5 21/30 Write block 440 len 24576 data: 5 5 5 4 2 3 22/30 Read block 191 len 4096 23/30 Read block 395 len 8192 24/30 Write block 254 len 20480 data: 2 2 2 3 3 25/30 Write block 404 len 4096 data: 6 26/30 Write block 505 len 4096 data: 7 27/30 Read block 500 len 12288 28/30 Read block 192 len 16384 29/30 Read block 586 len 36864 30/30 Read block 367 len 28672 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 2 Done Sent Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 8 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Write 5 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Read 3 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Read 7 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 0 28 Done 30 2 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:36.644902437Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:36.645192447Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1333,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 46 50 80 76 80 F F F <--- 1 020-039 50 45 50 80 73 80 F F F <--- 2 040-059 42 42 42 70 70 70 F F F 3 060-079 48 48 48 78 78 78 T F T <--- 4 080-099 48 48 48 78 78 78 F F F 5 100-119 48 48 48 78 78 78 F F F 6 120-139 45 45 45 73 73 73 F F F 7 140-159 48 48 48 78 78 78 F F F 8 160-179 47 47 47 77 77 77 F F F 9 180-199 46 46 46 74 74 74 F F F 10 200-219 48 48 48 78 78 78 F F F 11 220-239 50 42 50 80 69 80 F F F <--- 12 240-259 48 48 48 78 78 78 T F T <--- 13 260-279 48 48 48 78 78 78 F F F 14 280-299 48 48 48 78 78 78 F F F 15 300-319 45 45 45 73 73 73 T F T <--- 16 320-339 48 48 48 78 78 78 T F T <--- 17 340-359 50 48 50 80 78 80 F F F <--- 18 360-379 50 48 50 80 78 80 F T F <--- 19 380-399 45 45 45 73 73 73 F F F 20 400-419 45 45 45 73 73 73 T F T <--- 21 420-439 45 45 45 73 73 73 F F F 22 440-459 45 45 45 73 73 73 T F T <--- 23 460-479 45 45 45 73 73 73 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 50 40 50 80 66 80 T F T <--- 26 520-539 48 48 48 78 78 78 F F F 27 540-559 47 47 47 77 77 77 F F F 28 560-579 48 48 48 78 78 78 F F F 29 580-599 48 48 48 78 78 78 F F F Max gen: 50, Max flush: 80 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 80 80 80 F F F 1 020-039 50 50 50 80 80 80 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 51 51 51 81 81 81 F F F 4 080-099 48 48 48 78 78 78 F F F 5 100-119 48 48 48 78 78 78 F F F 6 120-139 45 45 45 73 73 73 F F F 7 140-159 48 48 48 78 78 78 F F F 8 160-179 47 47 47 77 77 77 F F F 9 180-199 46 46 46 74 74 74 F F F 10 200-219 48 48 48 78 78 78 F F F 11 220-239 50 50 50 80 80 80 F F F 12 240-259 51 51 51 81 81 81 F F F 13 260-279 48 48 48 78 78 78 F F F 14 280-299 48 48 48 78 78 78 F F F 15 300-319 51 51 51 81 81 81 F F F 16 320-339 51 51 51 81 81 81 F F F 17 340-359 50 50 50 80 80 80 F F F 18 360-379 50 50 50 80 80 80 F F F 19 380-399 45 45 45 73 73 73 F F F 20 400-419 51 51 51 81 81 81 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 51 51 51 81 81 81 F F F 23 460-479 45 45 45 73 73 73 F F F 24 480-499 36 36 36 59 59 59 F F F 25 500-519 51 51 51 81 81 81 F F F 26 520-539 48 48 48 78 78 78 F F F 27 540-559 47 47 47 77 77 77 F F F 28 560-579 48 48 48 78 78 78 F F F 29 580-599 48 48 48 78 78 78 F F F Max gen: 51, Max flush: 81 Begin loop 26 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.889820756Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:40.897010297Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.897099921Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.904299531Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.904566873Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.904590431Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.90459948Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.9046119Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341} {"msg":"Crucible 58534fc2-5826-4e16-8b16-62bed5b7d30a has session id: d698a1bf-84b4-46cc-8f06-a4c4be83b490","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.904939197Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"Upstairs opts: Upstairs UUID: 58534fc2-5826-4e16-8b16-62bed5b7d30a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.904987874Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"Crucible stats registered with UUID: 58534fc2-5826-4e16-8b16-62bed5b7d30a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.904998983Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:40.905017302Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905207739Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905239767Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905251266Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905352969Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905377638Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341} {"msg":"58534fc2-5826-4e16-8b16-62bed5b7d30a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905387117Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905396556Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"client":"0","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905406826Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"client":"1","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905416985Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"client":"2","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905426994Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905437383Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.90548103Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"ds_connection connected from Ok(127.0.0.1:51988)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905507369Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"io task","client":"0","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"ds_connection connected from Ok(127.0.0.1:34206)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905519668Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"io task","client":"2","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"ds_connection connected from Ok(127.0.0.1:59412)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.905530857Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"io task","client":"1","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906136806Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"client":"2","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906169054Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"client":"1","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906334532Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"client":"0","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906431126Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906450495Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"[0]R flush_numbers[0..12]: [80, 80, 70, 81, 78, 78, 73, 78, 77, 74, 78, 80]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906598494Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"downstairs","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"[0]R generation[0..12]: [50, 50, 42, 51, 48, 48, 45, 48, 47, 46, 48, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906609994Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"downstairs","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906621343Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"downstairs","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"[1]R flush_numbers[0..12]: [80, 80, 70, 81, 78, 78, 73, 78, 77, 74, 78, 80]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906632192Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"downstairs","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"[1]R generation[0..12]: [50, 50, 42, 51, 48, 48, 45, 48, 47, 46, 48, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906642911Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"downstairs","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906654131Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"downstairs","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"[2]R flush_numbers[0..12]: [80, 80, 70, 81, 78, 78, 73, 78, 77, 74, 78, 80]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.90666491Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"downstairs","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"[2]R generation[0..12]: [50, 50, 42, 51, 48, 48, 45, 48, 47, 46, 48, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906675849Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"downstairs","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906687108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"downstairs","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906697008Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"downstairs","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906707087Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"downstairs","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906718326Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"downstairs","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906728166Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"":"downstairs","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906738145Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906747314Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"client":"0","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906757184Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"client":"1","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906767143Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"client":"2","":"downstairs","session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906777452Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906786902Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"58534fc2-5826-4e16-8b16-62bed5b7d30a is now active with session: d698a1bf-84b4-46cc-8f06-a4c4be83b490","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906797191Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.90680643Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"session_id":"d698a1bf-84b4-46cc-8f06-a4c4be83b490"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.90681618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341} {"msg":"Activated sub_volume 58534fc2-5826-4e16-8b16-62bed5b7d30a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906824649Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.906832998Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.907184965Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"task":"crutest"} 01/30 Write block 304 len 36864 data: 6 6 7 7 6 5 3 5 6 02/30 Flush 03/30 Write block 255 len 40960 data: 3 3 4 4 4 3 4 4 3 3 04/30 Write block 170 len 28672 data: 11 10 8 9 9 7 5 05/30 Write block 104 len 28672 data: 6 4 4 5 7 6 6 06/30 Read block 322 len 8192 07/30 Read block 382 len 4096 08/30 Read block 155 len 24576 09/30 Write block 475 len 36864 data: 3 3 3 5 5 5 4 4 4 10/30 Write block 361 len 32768 data: 9 7 6 6 7 8 6 7 11/30 Read block 499 len 4096 12/30 Read block 270 len 4096 13/30 Read block 455 len 28672 14/30 Write block 167 len 32768 data: 10 9 10 12 11 9 10 10 15/30 Read block 28 len 24576 16/30 Read block 523 len 20480 17/30 Read block 172 len 12288 18/30 Read block 443 len 4096 19/30 Flush 20/30 Write block 30 len 20480 data: 5 6 4 7 6 21/30 Read block 36 len 20480 22/30 Write block 310 len 20480 data: 4 6 7 6 6 23/30 Write block 325 len 20480 data: 3 3 4 6 8 24/30 Write block 441 len 4096 data: 6 25/30 Read block 518 len 20480 26/30 Flush 27/30 Write block 309 len 40960 data: 6 5 7 8 7 7 5 7 9 9 28/30 Read block 465 len 8192 29/30 Write block 291 len 20480 data: 2 3 3 3 4 30/30 Read block 467 len 4096 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 10 Sent Done Done false 1003 Acked Write 7 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Read 2 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Write 9 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Read 5 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 0 28 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:40.916138595Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:40.916430176Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1341,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 80 80 80 F F F 1 020-039 50 52 52 80 84 84 F F F <--- 2 040-059 42 42 42 70 70 70 F F F 3 060-079 51 51 51 81 81 81 F F F 4 080-099 48 48 48 78 78 78 F F F 5 100-119 48 52 52 78 83 83 F F F <--- 6 120-139 45 45 45 73 73 73 F F F 7 140-159 48 48 48 78 78 78 F F F 8 160-179 47 52 52 77 83 83 F F F <--- 9 180-199 46 46 46 74 74 74 F F F 10 200-219 48 48 48 78 78 78 F F F 11 220-239 50 50 50 80 80 80 F F F 12 240-259 51 52 52 81 83 83 F F F <--- 13 260-279 48 52 52 78 83 83 F F F <--- 14 280-299 48 48 48 78 78 78 F T T <--- 15 300-319 52 52 52 82 84 84 F T T <--- 16 320-339 51 52 52 81 84 84 F F F <--- 17 340-359 50 50 50 80 80 80 F F F 18 360-379 50 52 52 80 83 83 F F F <--- 19 380-399 45 45 45 73 73 73 F F F 20 400-419 51 51 51 81 81 81 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 51 52 52 81 84 84 F F F <--- 23 460-479 45 52 52 73 83 83 F F F <--- 24 480-499 36 52 52 59 83 83 F F F <--- 25 500-519 51 51 51 81 81 81 F F F 26 520-539 48 48 48 78 78 78 F F F 27 540-559 47 47 47 77 77 77 F F F 28 560-579 48 48 48 78 78 78 F F F 29 580-599 48 48 48 78 78 78 F F F Max gen: 52, Max flush: 84 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 80 80 80 F F F 1 020-039 52 52 52 84 84 84 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 51 51 51 81 81 81 F F F 4 080-099 48 48 48 78 78 78 F F F 5 100-119 52 52 52 83 83 83 F F F 6 120-139 45 45 45 73 73 73 F F F 7 140-159 48 48 48 78 78 78 F F F 8 160-179 52 52 52 83 83 83 F F F 9 180-199 46 46 46 74 74 74 F F F 10 200-219 48 48 48 78 78 78 F F F 11 220-239 50 50 50 80 80 80 F F F 12 240-259 52 52 52 83 83 83 F F F 13 260-279 52 52 52 83 83 83 F F F 14 280-299 53 53 53 85 85 85 F F F 15 300-319 53 53 53 85 85 85 F F F 16 320-339 52 52 52 84 84 84 F F F 17 340-359 50 50 50 80 80 80 F F F 18 360-379 52 52 52 83 83 83 F F F 19 380-399 45 45 45 73 73 73 F F F 20 400-419 51 51 51 81 81 81 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 52 52 52 84 84 84 F F F 23 460-479 52 52 52 83 83 83 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 51 51 51 81 81 81 F F F 26 520-539 48 48 48 78 78 78 F F F 27 540-559 47 47 47 77 77 77 F F F 28 560-579 48 48 48 78 78 78 F F F 29 580-599 48 48 48 78 78 78 F F F Max gen: 53, Max flush: 85 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.166213635Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.174581855Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.174865546Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.174890594Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.174901833Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.174913883Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349} {"msg":"Crucible 7d77a950-8455-4461-8216-23bf64c13840 has session id: e207f743-b288-48d7-a33a-85e47c4e1099","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175232371Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"Upstairs opts: Upstairs UUID: 7d77a950-8455-4461-8216-23bf64c13840, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175253569Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"Crucible stats registered with UUID: 7d77a950-8455-4461-8216-23bf64c13840","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175263949Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:45.175276638Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175442157Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175458766Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175469375Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175596576Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175610965Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349} {"msg":"7d77a950-8455-4461-8216-23bf64c13840 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175644963Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175654812Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"client":"0","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175664941Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"client":"1","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175677401Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"client":"2","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.17568747Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175698489Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175708908Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"ds_connection connected from Ok(127.0.0.1:60008)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175754225Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"io task","client":"2","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"ds_connection connected from Ok(127.0.0.1:54238)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175781094Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"io task","client":"1","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"ds_connection connected from Ok(127.0.0.1:47819)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.175792763Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"io task","client":"0","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.176499445Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"client":"2","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.176516274Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"client":"0","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.176616117Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"client":"1","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.176835142Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.17686108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"[0]R flush_numbers[0..12]: [80, 84, 70, 81, 78, 83, 73, 78, 83, 74, 78, 80]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.176955124Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"downstairs","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"[0]R generation[0..12]: [50, 52, 42, 51, 48, 52, 45, 48, 52, 46, 48, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.176966743Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"downstairs","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.176978212Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"downstairs","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"[1]R flush_numbers[0..12]: [80, 84, 70, 81, 78, 83, 73, 78, 83, 74, 78, 80]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.176991291Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"downstairs","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"[1]R generation[0..12]: [50, 52, 42, 51, 48, 52, 45, 48, 52, 46, 48, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.17700222Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"downstairs","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.17701379Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"downstairs","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"[2]R flush_numbers[0..12]: [80, 84, 70, 81, 78, 83, 73, 78, 83, 74, 78, 80]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177024469Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"downstairs","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"[2]R generation[0..12]: [50, 52, 42, 51, 48, 52, 45, 48, 52, 46, 48, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177035168Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"downstairs","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177046397Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"downstairs","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177056197Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"downstairs","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177066226Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"downstairs","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177075915Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"downstairs","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177085735Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"":"downstairs","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177095984Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177105233Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"client":"0","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177115283Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"client":"1","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177125322Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"client":"2","":"downstairs","session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177135661Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177146581Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"7d77a950-8455-4461-8216-23bf64c13840 is now active with session: e207f743-b288-48d7-a33a-85e47c4e1099","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.17715701Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177166119Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"session_id":"e207f743-b288-48d7-a33a-85e47c4e1099"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177175469Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349} {"msg":"Activated sub_volume 7d77a950-8455-4461-8216-23bf64c13840","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177183858Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177192398Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.177481918Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"task":"crutest"} 01/30 Write block 577 len 16384 data: 8 8 7 7 02/30 Flush 03/30 Write block 469 len 20480 data: 6 6 5 4 3 04/30 Write block 215 len 8192 data: 5 5 05/30 Read block 449 len 12288 06/30 Flush 07/30 Write block 508 len 12288 data: 7 7 6 08/30 Write block 4 len 40960 data: 4 5 6 6 5 5 5 4 4 5 09/30 Flush 10/30 Write block 513 len 28672 data: 5 5 4 4 2 3 4 11/30 Write block 93 len 28672 data: 7 6 6 6 7 7 7 12/30 Read block 142 len 16384 13/30 Flush 14/30 Flush 15/30 Write block 195 len 4096 data: 5 16/30 Read block 437 len 40960 17/30 Flush 18/30 Flush 19/30 Write block 98 len 16384 data: 8 8 5 4 20/30 Write block 571 len 24576 data: 5 5 6 6 9 8 21/30 Flush 22/30 Write block 361 len 16384 data: 10 8 7 7 23/30 Write block 395 len 36864 data: 6 6 7 7 7 5 6 6 5 24/30 Write block 139 len 8192 data: 5 5 25/30 Write block 199 len 4096 data: 4 26/30 Read block 312 len 4096 27/30 Read block 103 len 32768 28/30 Write block 473 len 8192 data: 4 3 29/30 Read block 169 len 32768 30/30 Write block 62 len 36864 data: 7 5 5 4 4 4 3 4 3 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 1 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 16 1 18 Done 15 0 15 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1013)) Some(JobId(1020)) Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:45.185282217Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:45.185565568Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1349,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 88 88 88 F F F 1 020-039 52 52 52 84 84 84 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 51 51 51 81 81 81 T F T <--- 4 080-099 54 54 54 93 89 93 F F F <--- 5 100-119 54 52 54 93 83 93 F F F <--- 6 120-139 45 45 45 73 73 73 T F T <--- 7 140-159 48 48 48 78 78 78 T F T <--- 8 160-179 52 52 52 83 83 83 F F F 9 180-199 54 46 54 91 74 91 T F T <--- 10 200-219 54 54 54 87 87 87 F F F 11 220-239 50 50 50 80 80 80 F F F 12 240-259 52 52 52 83 83 83 F F F 13 260-279 52 52 52 83 83 83 F F F 14 280-299 53 53 53 85 85 85 F F F 15 300-319 53 53 53 85 85 85 F F F 16 320-339 52 52 52 84 84 84 F F F 17 340-359 50 50 50 80 80 80 F F F 18 360-379 52 52 52 83 83 83 T F T <--- 19 380-399 45 45 45 73 73 73 T F T <--- 20 400-419 51 51 51 81 81 81 T F T <--- 21 420-439 45 45 45 73 73 73 F F F 22 440-459 52 52 52 84 84 84 F F F 23 460-479 54 54 54 87 87 87 T F T <--- 24 480-499 52 52 52 83 83 83 F F F 25 500-519 54 54 54 89 89 89 F F F 26 520-539 48 48 48 78 78 78 F F F 27 540-559 47 47 47 77 77 77 F F F 28 560-579 54 54 54 93 86 93 F F F <--- 29 580-599 54 54 54 86 86 86 F F F Max gen: 54, Max flush: 93 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 88 88 88 F F F 1 020-039 52 52 52 84 84 84 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 55 55 55 94 94 94 F F F 4 080-099 54 54 54 93 93 93 F F F 5 100-119 54 54 54 93 93 93 F F F 6 120-139 55 55 55 94 94 94 F F F 7 140-159 55 55 55 94 94 94 F F F 8 160-179 52 52 52 83 83 83 F F F 9 180-199 55 55 55 94 94 94 F F F 10 200-219 54 54 54 87 87 87 F F F 11 220-239 50 50 50 80 80 80 F F F 12 240-259 52 52 52 83 83 83 F F F 13 260-279 52 52 52 83 83 83 F F F 14 280-299 53 53 53 85 85 85 F F F 15 300-319 53 53 53 85 85 85 F F F 16 320-339 52 52 52 84 84 84 F F F 17 340-359 50 50 50 80 80 80 F F F 18 360-379 55 55 55 94 94 94 F F F 19 380-399 55 55 55 94 94 94 F F F 20 400-419 55 55 55 94 94 94 F F F 21 420-439 45 45 45 73 73 73 F F F 22 440-459 52 52 52 84 84 84 F F F 23 460-479 55 55 55 94 94 94 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 54 54 54 89 89 89 F F F 26 520-539 48 48 48 78 78 78 F F F 27 540-559 47 47 47 77 77 77 F F F 28 560-579 54 54 54 93 93 93 F F F 29 580-599 54 54 54 86 86 86 F F F Max gen: 55, Max flush: 94 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.426985916Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.435113363Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.435391564Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.435420802Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.435429901Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.435442021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357} {"msg":"Crucible a19ab431-696d-4f3e-a28b-49f11cad7b20 has session id: 2b40d0f8-cc32-4254-b62b-0fdd00a38857","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.435780777Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"Upstairs opts: Upstairs UUID: a19ab431-696d-4f3e-a28b-49f11cad7b20, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.435812735Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"Crucible stats registered with UUID: a19ab431-696d-4f3e-a28b-49f11cad7b20","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.435824245Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:49.435834014Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.435989713Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436009352Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436020201Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436194749Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436214178Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357} {"msg":"a19ab431-696d-4f3e-a28b-49f11cad7b20 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436251325Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436261105Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"client":"0","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436271374Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"client":"1","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436281423Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"client":"2","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436291543Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436302042Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436312531Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"ds_connection connected from Ok(127.0.0.1:40461)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436397236Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"io task","client":"2","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"ds_connection connected from Ok(127.0.0.1:47644)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436408725Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"io task","client":"0","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"ds_connection connected from Ok(127.0.0.1:56263)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.436500399Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"io task","client":"1","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437099278Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"client":"0","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437147495Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"client":"1","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437270336Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"client":"2","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437400817Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437420876Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"[0]R flush_numbers[0..12]: [88, 84, 70, 94, 93, 93, 94, 94, 83, 94, 87, 80]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437533768Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"downstairs","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"[0]R generation[0..12]: [54, 52, 42, 55, 54, 54, 55, 55, 52, 55, 54, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437545447Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"downstairs","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437557057Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"downstairs","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"[1]R flush_numbers[0..12]: [88, 84, 70, 94, 93, 93, 94, 94, 83, 94, 87, 80]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437567926Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"downstairs","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"[1]R generation[0..12]: [54, 52, 42, 55, 54, 54, 55, 55, 52, 55, 54, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437578565Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"downstairs","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437589914Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"downstairs","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"[2]R flush_numbers[0..12]: [88, 84, 70, 94, 93, 93, 94, 94, 83, 94, 87, 80]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437600734Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"downstairs","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"[2]R generation[0..12]: [54, 52, 42, 55, 54, 54, 55, 55, 52, 55, 54, 50]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437611413Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"downstairs","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437622562Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"downstairs","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437632372Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"downstairs","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437642391Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"downstairs","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.43765355Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"downstairs","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437663649Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"":"downstairs","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437673749Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437687818Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"client":"0","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437697997Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"client":"1","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437707946Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"client":"2","":"downstairs","session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437718146Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437727515Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"a19ab431-696d-4f3e-a28b-49f11cad7b20 is now active with session: 2b40d0f8-cc32-4254-b62b-0fdd00a38857","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437737834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437747354Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437755623Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"session_id":"2b40d0f8-cc32-4254-b62b-0fdd00a38857"} {"msg":"Activated sub_volume a19ab431-696d-4f3e-a28b-49f11cad7b20","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437765282Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.437773702Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.438107849Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"task":"crutest"} 01/30 Write block 263 len 28672 data: 4 4 3 3 2 4 4 02/30 Flush 03/30 Write block 575 len 32768 data: 10 9 9 9 8 8 7 7 04/30 Flush 05/30 Write block 323 len 40960 data: 7 7 4 4 5 7 9 8 8 8 06/30 Write block 224 len 20480 data: 7 4 5 5 6 07/30 Read block 130 len 12288 08/30 Read block 34 len 32768 09/30 Write block 331 len 16384 data: 9 9 8 6 10/30 Flush 11/30 Write block 374 len 40960 data: 6 8 7 7 8 6 5 5 5 4 12/30 Write block 226 len 12288 data: 6 6 7 13/30 Read block 480 len 24576 14/30 Write block 419 len 16384 data: 6 5 5 5 15/30 Read block 88 len 32768 16/30 Read block 397 len 8192 17/30 Write block 390 len 8192 data: 3 4 18/30 Read block 517 len 12288 19/30 Write block 63 len 12288 data: 6 6 5 20/30 Read block 342 len 24576 21/30 Read block 306 len 40960 22/30 Flush 23/30 Write block 202 len 4096 data: 6 24/30 Write block 500 len 32768 data: 3 4 5 5 6 8 6 8 25/30 Flush 26/30 Write block 567 len 28672 data: 6 7 9 8 6 6 7 27/30 Write block 572 len 36864 data: 7 8 7 11 10 10 10 9 9 28/30 Write block 98 len 24576 data: 9 9 6 5 4 5 29/30 Write block 342 len 36864 data: 4 4 5 5 5 4 3 5 5 30/30 Write block 121 len 28672 data: 3 3 4 5 7 7 5 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 8 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Write 3 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Write 4 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Write 2 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 7 Done Sent Sent false 1026 Acked Write 9 Sent Sent Sent false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 5 28 37 Done 24 23 0 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:49.446567834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:49.446869083Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1357,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 88 88 88 F F F 1 020-039 52 52 52 84 84 84 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 56 56 55 98 98 94 F F F <--- 4 080-099 54 54 54 93 93 93 T T F <--- 5 100-119 54 54 54 93 93 93 T T F <--- 6 120-139 55 55 55 94 94 94 T T F <--- 7 140-159 55 55 55 94 94 94 F F F 8 160-179 52 52 52 83 83 83 F F F 9 180-199 55 55 55 94 94 94 F F F 10 200-219 56 56 54 99 99 87 F F F <--- 11 220-239 56 56 50 98 98 80 F F F <--- 12 240-259 52 52 52 83 83 83 F F F 13 260-279 56 56 56 95 95 95 F F F 14 280-299 53 53 53 85 85 85 F F F 15 300-319 53 53 53 85 85 85 F F F 16 320-339 56 56 52 97 97 84 F F F <--- 17 340-359 50 50 50 80 80 80 T T F <--- 18 360-379 56 56 55 98 98 94 F F F <--- 19 380-399 56 56 55 98 98 94 F F F <--- 20 400-419 56 56 55 98 98 94 F F F <--- 21 420-439 56 56 45 98 98 73 F F F <--- 22 440-459 52 52 52 84 84 84 F F F 23 460-479 55 55 55 94 94 94 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 56 56 54 99 99 89 F F F <--- 26 520-539 48 48 48 78 78 78 F F F 27 540-559 47 47 47 77 77 77 F F F 28 560-579 56 56 54 96 96 93 T T F <--- 29 580-599 56 56 54 96 96 86 T T F <--- Max gen: 56, Max flush: 99 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 88 88 88 F F F 1 020-039 52 52 52 84 84 84 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 56 56 56 98 98 98 F F F 4 080-099 57 57 57 100 100 100 F F F 5 100-119 57 57 57 100 100 100 F F F 6 120-139 57 57 57 100 100 100 F F F 7 140-159 55 55 55 94 94 94 F F F 8 160-179 52 52 52 83 83 83 F F F 9 180-199 55 55 55 94 94 94 F F F 10 200-219 56 56 56 99 99 99 F F F 11 220-239 56 56 56 98 98 98 F F F 12 240-259 52 52 52 83 83 83 F F F 13 260-279 56 56 56 95 95 95 F F F 14 280-299 53 53 53 85 85 85 F F F 15 300-319 53 53 53 85 85 85 F F F 16 320-339 56 56 56 97 97 97 F F F 17 340-359 57 57 57 100 100 100 F F F 18 360-379 56 56 56 98 98 98 F F F 19 380-399 56 56 56 98 98 98 F F F 20 400-419 56 56 56 98 98 98 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 52 52 52 84 84 84 F F F 23 460-479 55 55 55 94 94 94 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 56 56 56 99 99 99 F F F 26 520-539 48 48 48 78 78 78 F F F 27 540-559 47 47 47 77 77 77 F F F 28 560-579 57 57 57 100 100 100 F F F 29 580-599 57 57 57 100 100 100 F F F Max gen: 57, Max flush: 100 Begin loop 29 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.695165453Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.703184018Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.703463909Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.703494017Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.703506156Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.703518145Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365} {"msg":"Crucible 2831773f-d273-433a-bac9-280a22f6bd6c has session id: 0b2bffbf-2060-4164-b2b8-56003847cb19","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.703804645Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"Upstairs opts: Upstairs UUID: 2831773f-d273-433a-bac9-280a22f6bd6c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.703835833Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"Crucible stats registered with UUID: 2831773f-d273-433a-bac9-280a22f6bd6c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.703847093Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:53.703856812Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704022221Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704039919Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704050519Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704167721Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704194639Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365} {"msg":"2831773f-d273-433a-bac9-280a22f6bd6c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704204588Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704214178Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"client":"0","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704224667Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"client":"1","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704236896Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"client":"2","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704246885Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704257365Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704267724Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"ds_connection connected from Ok(127.0.0.1:48827)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704393055Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"io task","client":"1","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"ds_connection connected from Ok(127.0.0.1:34927)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.704447802Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"io task","client":"2","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"ds_connection connected from Ok(127.0.0.1:36952)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.7044727Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"io task","client":"0","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705025282Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"client":"0","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705050371Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"client":"2","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.70506617Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"client":"1","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705323052Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.70535509Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"[0]R flush_numbers[0..12]: [88, 84, 70, 98, 100, 100, 100, 94, 83, 94, 99, 98]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705416826Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"downstairs","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"[0]R generation[0..12]: [54, 52, 42, 56, 57, 57, 57, 55, 52, 55, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705428335Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"downstairs","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705439784Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"downstairs","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"[1]R flush_numbers[0..12]: [88, 84, 70, 98, 100, 100, 100, 94, 83, 94, 99, 98]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705453223Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"downstairs","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"[1]R generation[0..12]: [54, 52, 42, 56, 57, 57, 57, 55, 52, 55, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705463943Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"downstairs","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705475262Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"downstairs","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"[2]R flush_numbers[0..12]: [88, 84, 70, 98, 100, 100, 100, 94, 83, 94, 99, 98]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705486171Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"downstairs","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"[2]R generation[0..12]: [54, 52, 42, 56, 57, 57, 57, 55, 52, 55, 56, 56]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.7054968Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"downstairs","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.7055083Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"downstairs","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705518159Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"downstairs","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705528158Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"downstairs","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705537838Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"downstairs","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705547687Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"":"downstairs","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705557826Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705566876Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"client":"0","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705576785Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"client":"1","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705586614Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"client":"2","":"downstairs","session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705596804Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705607983Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"2831773f-d273-433a-bac9-280a22f6bd6c is now active with session: 0b2bffbf-2060-4164-b2b8-56003847cb19","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705618142Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705627191Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"session_id":"0b2bffbf-2060-4164-b2b8-56003847cb19"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705636531Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365} {"msg":"Activated sub_volume 2831773f-d273-433a-bac9-280a22f6bd6c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.70564501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.70565341Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.705982557Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"task":"crutest"} 01/30 Write block 409 len 16384 data: 3 3 4 4 02/30 Read block 386 len 8192 03/30 Write block 150 len 12288 data: 5 5 4 04/30 Read block 84 len 28672 05/30 Write block 200 len 8192 data: 6 6 06/30 Write block 260 len 4096 data: 4 07/30 Write block 144 len 36864 data: 6 4 5 6 5 5 6 6 5 08/30 Read block 277 len 20480 09/30 Read block 542 len 16384 10/30 Read block 0 len 16384 11/30 Read block 110 len 36864 12/30 Read block 376 len 20480 13/30 Write block 307 len 40960 data: 8 7 7 6 8 9 8 8 6 8 14/30 Write block 520 len 16384 data: 4 4 5 4 15/30 Flush 16/30 Write block 548 len 20480 data: 6 6 5 4 4 17/30 Write block 239 len 24576 data: 4 4 5 7 7 7 18/30 Write block 383 len 12288 data: 5 2 2 19/30 Write block 301 len 28672 data: 5 5 5 7 7 8 9 20/30 Read block 25 len 28672 21/30 Read block 263 len 28672 22/30 Write block 22 len 40960 data: 5 3 2 3 3 4 5 4 6 7 23/30 Write block 352 len 40960 data: 5 6 5 6 7 7 8 9 8 11 24/30 Read block 78 len 24576 25/30 Write block 525 len 32768 data: 5 5 5 4 5 5 5 6 26/30 Read block 531 len 20480 27/30 Read block 517 len 36864 28/30 Read block 69 len 32768 29/30 Write block 307 len 40960 data: 10 8 8 7 9 10 9 9 7 9 30/30 Read block 89 len 4096 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Sent Done false 1001 Acked Read 2 Done Sent Done false 1002 Acked Write 3 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Read 5 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Read 9 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Write 10 Done Sent Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:53.714547685Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:53.714826816Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1365,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 88 88 88 F F F 1 020-039 52 52 52 84 84 84 T F T <--- 2 040-059 42 42 42 70 70 70 F F F 3 060-079 56 56 56 98 98 98 F F F 4 080-099 57 57 57 100 100 100 F F F 5 100-119 57 57 57 100 100 100 F F F 6 120-139 57 57 57 100 100 100 F F F 7 140-159 58 55 58 101 94 101 F F F <--- 8 160-179 52 52 52 83 83 83 F F F 9 180-199 55 55 55 94 94 94 F F F 10 200-219 58 56 58 101 99 101 F F F <--- 11 220-239 56 56 56 98 98 98 T F T <--- 12 240-259 52 52 52 83 83 83 T F T <--- 13 260-279 58 56 58 101 95 101 F F F <--- 14 280-299 53 53 53 85 85 85 F F F 15 300-319 58 53 58 101 85 101 T F T <--- 16 320-339 56 56 56 97 97 97 F F F 17 340-359 57 57 57 100 100 100 T F T <--- 18 360-379 56 56 56 98 98 98 T F T <--- 19 380-399 56 56 56 98 98 98 T F T <--- 20 400-419 58 56 58 101 98 101 F F F <--- 21 420-439 56 56 56 98 98 98 F F F 22 440-459 52 52 52 84 84 84 F F F 23 460-479 55 55 55 94 94 94 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 56 56 56 99 99 99 F F F 26 520-539 58 48 58 101 78 101 T F T <--- 27 540-559 47 47 47 77 77 77 T F T <--- 28 560-579 57 57 57 100 100 100 F F F 29 580-599 57 57 57 100 100 100 F F F Max gen: 58, Max flush: 101 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 54 54 54 88 88 88 F F F 1 020-039 59 59 59 102 102 102 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 56 56 56 98 98 98 F F F 4 080-099 57 57 57 100 100 100 F F F 5 100-119 57 57 57 100 100 100 F F F 6 120-139 57 57 57 100 100 100 F F F 7 140-159 58 58 58 101 101 101 F F F 8 160-179 52 52 52 83 83 83 F F F 9 180-199 55 55 55 94 94 94 F F F 10 200-219 58 58 58 101 101 101 F F F 11 220-239 59 59 59 102 102 102 F F F 12 240-259 59 59 59 102 102 102 F F F 13 260-279 58 58 58 101 101 101 F F F 14 280-299 53 53 53 85 85 85 F F F 15 300-319 59 59 59 102 102 102 F F F 16 320-339 56 56 56 97 97 97 F F F 17 340-359 59 59 59 102 102 102 F F F 18 360-379 59 59 59 102 102 102 F F F 19 380-399 59 59 59 102 102 102 F F F 20 400-419 58 58 58 101 101 101 F F F 21 420-439 56 56 56 98 98 98 F F F 22 440-459 52 52 52 84 84 84 F F F 23 460-479 55 55 55 94 94 94 F F F 24 480-499 52 52 52 83 83 83 F F F 25 500-519 56 56 56 99 99 99 F F F 26 520-539 59 59 59 102 102 102 F F F 27 540-559 59 59 59 102 102 102 F F F 28 560-579 57 57 57 100 100 100 F F F 29 580-599 57 57 57 100 100 100 F F F Max gen: 59, Max flush: 102 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.960843931Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.968964088Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.969251249Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.969278067Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.969287026Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.969298935Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373} {"msg":"Crucible d00720b9-56cd-43a7-9274-1fe2867d689c has session id: 7dbef66f-a250-4ed5-a261-b3fd0aca5f0d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.969696288Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"Upstairs opts: Upstairs UUID: d00720b9-56cd-43a7-9274-1fe2867d689c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.969715177Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"Crucible stats registered with UUID: d00720b9-56cd-43a7-9274-1fe2867d689c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.969727516Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:57.969739785Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.969908684Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.969940432Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.969951901Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.970050044Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.970075323Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373} {"msg":"d00720b9-56cd-43a7-9274-1fe2867d689c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.970087672Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.970096951Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"client":"0","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.97010685Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"client":"1","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.97011727Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"client":"2","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.970127179Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.970137728Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.970147998Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56557)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.970278529Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"io task","client":"2","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57486)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.970326775Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"io task","client":"0","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62297)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.970340934Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"io task","client":"1","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.970813022Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"client":"2","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.970925925Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"client":"0","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.970958992Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"client":"1","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.97113582Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971176578Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"[0]R flush_numbers[0..12]: [88, 102, 70, 98, 100, 100, 100, 101, 83, 94, 101, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971226904Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"downstairs","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"[0]R generation[0..12]: [54, 59, 42, 56, 57, 57, 57, 58, 52, 55, 58, 59]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971238553Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"downstairs","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971249963Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"downstairs","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"[1]R flush_numbers[0..12]: [88, 102, 70, 98, 100, 100, 100, 101, 83, 94, 101, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971260942Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"downstairs","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"[1]R generation[0..12]: [54, 59, 42, 56, 57, 57, 57, 58, 52, 55, 58, 59]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971271651Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"downstairs","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.97128284Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"downstairs","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"[2]R flush_numbers[0..12]: [88, 102, 70, 98, 100, 100, 100, 101, 83, 94, 101, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.97129361Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"downstairs","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"[2]R generation[0..12]: [54, 59, 42, 56, 57, 57, 57, 58, 52, 55, 58, 59]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971304299Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"downstairs","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971317898Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"downstairs","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971327557Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"downstairs","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971337707Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"downstairs","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971347416Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"downstairs","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971357115Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"":"downstairs","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971367305Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971376364Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"client":"0","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971386463Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"client":"1","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971396353Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"client":"2","":"downstairs","session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971406382Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971415771Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"d00720b9-56cd-43a7-9274-1fe2867d689c is now active with session: 7dbef66f-a250-4ed5-a261-b3fd0aca5f0d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971426011Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.97143545Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971443339Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"session_id":"7dbef66f-a250-4ed5-a261-b3fd0aca5f0d"} {"msg":"Activated sub_volume d00720b9-56cd-43a7-9274-1fe2867d689c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971452659Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971461148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.971768807Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"task":"crutest"} 01/30 Write block 149 len 24576 data: 6 7 7 6 3 3 02/30 Write block 411 len 32768 data: 5 5 5 6 8 8 6 6 03/30 Read block 501 len 24576 04/30 Write block 504 len 32768 data: 7 9 7 9 8 8 7 6 05/30 Write block 87 len 36864 data: 3 3 2 2 3 6 8 7 7 06/30 Read block 408 len 24576 07/30 Write block 487 len 32768 data: 6 5 5 5 4 4 5 3 08/30 Read block 158 len 36864 09/30 Flush 10/30 Write block 154 len 16384 data: 4 3 4 4 11/30 Write block 379 len 32768 data: 7 6 6 6 6 3 3 2 12/30 Write block 246 len 8192 data: 7 6 13/30 Write block 323 len 40960 data: 8 8 5 5 6 8 10 9 10 10 14/30 Flush 15/30 Write block 393 len 32768 data: 4 6 7 7 8 8 8 6 16/30 Read block 68 len 36864 17/30 Read block 413 len 24576 18/30 Read block 300 len 24576 19/30 Write block 0 len 32768 data: 2 3 3 3 5 6 7 7 20/30 Write block 377 len 24576 data: 8 9 8 7 7 7 21/30 Write block 132 len 32768 data: 5 3 4 3 4 5 5 6 22/30 Read block 231 len 40960 23/30 Flush 24/30 Write block 393 len 12288 data: 5 7 8 25/30 Write block 329 len 8192 data: 11 10 26/30 Write block 465 len 36864 data: 9 9 6 6 7 7 6 5 5 27/30 Read block 61 len 12288 28/30 Read block 35 len 12288 29/30 Read block 196 len 16384 30/30 Write block 413 len 32768 data: 6 7 9 9 7 7 7 6 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 6 Done Sent Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Read 6 Done Sent Done false 1006 Acked Write 8 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Write 2 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Write 8 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 3 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:40:57.980680151Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:40:57.981017018Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1373,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 54 60 105 88 105 F F F <--- 1 020-039 59 59 59 102 102 102 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 56 56 56 98 98 98 F F F 4 080-099 60 57 60 103 100 103 F F F <--- 5 100-119 57 57 57 100 100 100 F F F 6 120-139 60 57 60 105 100 105 F F F <--- 7 140-159 60 58 60 104 101 104 F T F <--- 8 160-179 52 52 52 83 83 83 F F F 9 180-199 55 55 55 94 94 94 F F F 10 200-219 58 58 58 101 101 101 F F F 11 220-239 59 59 59 102 102 102 F F F 12 240-259 60 59 60 104 102 104 F F F <--- 13 260-279 58 58 58 101 101 101 F F F 14 280-299 53 53 53 85 85 85 F F F 15 300-319 59 59 59 102 102 102 F F F 16 320-339 60 56 60 104 97 104 T F T <--- 17 340-359 59 59 59 102 102 102 F F F 18 360-379 60 59 60 105 102 105 F F F <--- 19 380-399 60 59 60 105 102 105 T F T <--- 20 400-419 60 58 60 105 101 105 T T T <--- 21 420-439 56 56 56 98 98 98 T F T <--- 22 440-459 52 52 52 84 84 84 F F F 23 460-479 55 55 55 94 94 94 T F T <--- 24 480-499 60 52 60 103 83 103 F F F <--- 25 500-519 60 56 60 103 99 103 F F F <--- 26 520-539 59 59 59 102 102 102 F F F 27 540-559 59 59 59 102 102 102 F F F 28 560-579 57 57 57 100 100 100 F F F 29 580-599 57 57 57 100 100 100 F F F Max gen: 60, Max flush: 105 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 105 105 105 F F F 1 020-039 59 59 59 102 102 102 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 56 56 56 98 98 98 F F F 4 080-099 60 60 60 103 103 103 F F F 5 100-119 57 57 57 100 100 100 F F F 6 120-139 60 60 60 105 105 105 F F F 7 140-159 60 60 60 104 104 104 F F F 8 160-179 52 52 52 83 83 83 F F F 9 180-199 55 55 55 94 94 94 F F F 10 200-219 58 58 58 101 101 101 F F F 11 220-239 59 59 59 102 102 102 F F F 12 240-259 60 60 60 104 104 104 F F F 13 260-279 58 58 58 101 101 101 F F F 14 280-299 53 53 53 85 85 85 F F F 15 300-319 59 59 59 102 102 102 F F F 16 320-339 61 61 61 106 106 106 F F F 17 340-359 59 59 59 102 102 102 F F F 18 360-379 60 60 60 105 105 105 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 61 61 61 106 106 106 F F F 22 440-459 52 52 52 84 84 84 F F F 23 460-479 61 61 61 106 106 106 F F F 24 480-499 60 60 60 103 103 103 F F F 25 500-519 60 60 60 103 103 103 F F F 26 520-539 59 59 59 102 102 102 F F F 27 540-559 59 59 59 102 102 102 F F F 28 560-579 57 57 57 100 100 100 F F F 29 580-599 57 57 57 100 100 100 F F F Max gen: 61, Max flush: 106 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.22561831Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:02.232742725Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.23281397Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.239735369Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.239995471Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240031329Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240041088Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240053158Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381} {"msg":"Crucible 914b7d8f-b256-427a-8358-040d06862033 has session id: 59ab483b-71b4-445f-87f1-52ec4a478eab","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240375706Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"Upstairs opts: Upstairs UUID: 914b7d8f-b256-427a-8358-040d06862033, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240406294Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"Crucible stats registered with UUID: 914b7d8f-b256-427a-8358-040d06862033","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240418043Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:02.240427762Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240590461Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.24061254Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240623439Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240769739Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240795187Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381} {"msg":"914b7d8f-b256-427a-8358-040d06862033 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240819445Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240829215Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"client":"0","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240839354Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"client":"1","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240849553Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"client":"2","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240859583Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240869992Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240880441Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"ds_connection connected from Ok(127.0.0.1:35297)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240955226Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"io task","client":"1","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"ds_connection connected from Ok(127.0.0.1:58909)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.240981224Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"io task","client":"2","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"ds_connection connected from Ok(127.0.0.1:50403)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.24104217Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"io task","client":"0","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.241676517Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"client":"1","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.241699446Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"client":"2","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.241712995Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"client":"0","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242018334Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242028783Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"[0]R flush_numbers[0..12]: [105, 102, 70, 98, 103, 100, 105, 104, 83, 94, 101, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242064571Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"downstairs","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"[0]R generation[0..12]: [60, 59, 42, 56, 60, 57, 60, 60, 52, 55, 58, 59]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.24207596Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"downstairs","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242087669Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"downstairs","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"[1]R flush_numbers[0..12]: [105, 102, 70, 98, 103, 100, 105, 104, 83, 94, 101, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242098588Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"downstairs","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"[1]R generation[0..12]: [60, 59, 42, 56, 60, 57, 60, 60, 52, 55, 58, 59]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242109258Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"downstairs","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242120327Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"downstairs","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"[2]R flush_numbers[0..12]: [105, 102, 70, 98, 103, 100, 105, 104, 83, 94, 101, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242131206Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"downstairs","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"[2]R generation[0..12]: [60, 59, 42, 56, 60, 57, 60, 60, 52, 55, 58, 59]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242141805Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"downstairs","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242152865Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"downstairs","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242162654Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"downstairs","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242172523Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"downstairs","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242183843Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"downstairs","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242193512Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"":"downstairs","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242203671Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242212741Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"client":"0","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.24222272Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"client":"1","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242232789Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"client":"2","":"downstairs","session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242242999Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242252508Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"914b7d8f-b256-427a-8358-040d06862033 is now active with session: 59ab483b-71b4-445f-87f1-52ec4a478eab","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242262767Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242272267Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242280226Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"session_id":"59ab483b-71b4-445f-87f1-52ec4a478eab"} {"msg":"Activated sub_volume 914b7d8f-b256-427a-8358-040d06862033","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242289755Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242298175Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.242601334Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"task":"crutest"} 01/30 Write block 71 len 8192 data: 4 6 02/30 Read block 223 len 20480 03/30 Flush 04/30 Flush 05/30 Write block 204 len 8192 data: 8 7 06/30 Write block 337 len 32768 data: 7 6 6 5 5 5 5 6 07/30 Write block 364 len 12288 data: 8 8 9 08/30 Write block 208 len 12288 data: 7 5 6 09/30 Read block 469 len 16384 10/30 Write block 487 len 16384 data: 7 6 6 6 11/30 Write block 317 len 16384 data: 10 10 10 9 12/30 Read block 315 len 36864 13/30 Read block 461 len 36864 14/30 Read block 119 len 20480 15/30 Read block 119 len 16384 16/30 Write block 10 len 16384 data: 6 5 5 6 17/30 Write block 30 len 16384 data: 7 8 5 8 18/30 Write block 485 len 12288 data: 5 6 8 19/30 Flush 20/30 Write block 29 len 28672 data: 5 8 9 6 9 7 6 21/30 Flush 22/30 Flush 23/30 Write block 585 len 16384 data: 6 5 6 4 24/30 Read block 581 len 20480 25/30 Flush 26/30 Write block 140 len 40960 data: 6 7 7 7 7 5 6 7 6 7 27/30 Write block 449 len 8192 data: 5 5 28/30 Read block 160 len 4096 29/30 Read block 393 len 12288 30/30 Read block 281 len 20480 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 7 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 0 27 Done 0 27 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:02.250571052Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:02.250854943Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1381,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 62 62 105 109 109 F F F <--- 1 020-039 59 62 62 102 110 110 F F F <--- 2 040-059 42 42 42 70 70 70 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 60 60 60 103 103 103 F F F 5 100-119 57 57 57 100 100 100 F F F 6 120-139 60 60 60 105 105 105 F F F 7 140-159 60 60 60 104 104 104 F T T <--- 8 160-179 52 52 52 83 83 83 F F F 9 180-199 55 55 55 94 94 94 F F F 10 200-219 58 62 62 101 109 109 F F F <--- 11 220-239 59 59 59 102 102 102 F F F 12 240-259 60 60 60 104 104 104 F F F 13 260-279 58 58 58 101 101 101 F F F 14 280-299 53 53 53 85 85 85 F F F 15 300-319 59 62 62 102 109 109 F F F <--- 16 320-339 61 62 62 106 109 109 F F F <--- 17 340-359 59 62 62 102 109 109 F F F <--- 18 360-379 60 62 62 105 109 109 F F F <--- 19 380-399 61 61 61 106 106 106 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 61 61 61 106 106 106 F F F 22 440-459 52 52 52 84 84 84 F T T <--- 23 460-479 61 61 61 106 106 106 F F F 24 480-499 60 62 62 103 109 109 F F F <--- 25 500-519 60 60 60 103 103 103 F F F 26 520-539 59 59 59 102 102 102 F F F 27 540-559 59 59 59 102 102 102 F F F 28 560-579 57 57 57 100 100 100 F F F 29 580-599 57 62 62 100 112 112 F F F <--- Max gen: 62, Max flush: 112 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 109 109 109 F F F 1 020-039 62 62 62 110 110 110 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 60 60 60 103 103 103 F F F 5 100-119 57 57 57 100 100 100 F F F 6 120-139 60 60 60 105 105 105 F F F 7 140-159 63 63 63 113 113 113 F F F 8 160-179 52 52 52 83 83 83 F F F 9 180-199 55 55 55 94 94 94 F F F 10 200-219 62 62 62 109 109 109 F F F 11 220-239 59 59 59 102 102 102 F F F 12 240-259 60 60 60 104 104 104 F F F 13 260-279 58 58 58 101 101 101 F F F 14 280-299 53 53 53 85 85 85 F F F 15 300-319 62 62 62 109 109 109 F F F 16 320-339 62 62 62 109 109 109 F F F 17 340-359 62 62 62 109 109 109 F F F 18 360-379 62 62 62 109 109 109 F F F 19 380-399 61 61 61 106 106 106 F F F 20 400-419 61 61 61 106 106 106 F F F 21 420-439 61 61 61 106 106 106 F F F 22 440-459 63 63 63 113 113 113 F F F 23 460-479 61 61 61 106 106 106 F F F 24 480-499 62 62 62 109 109 109 F F F 25 500-519 60 60 60 103 103 103 F F F 26 520-539 59 59 59 102 102 102 F F F 27 540-559 59 59 59 102 102 102 F F F 28 560-579 57 57 57 100 100 100 F F F 29 580-599 62 62 62 112 112 112 F F F Max gen: 63, Max flush: 113 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.493445841Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.5018446Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502122481Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502148579Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502159848Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502171747Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389} {"msg":"Crucible de202393-c9de-4575-ada9-b27915d1c9ed has session id: 2b395523-dce6-47bf-acaf-e587a16bd36d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502502855Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"Upstairs opts: Upstairs UUID: de202393-c9de-4575-ada9-b27915d1c9ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502533613Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"Crucible stats registered with UUID: de202393-c9de-4575-ada9-b27915d1c9ed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502545162Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:06.502559441Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.50272638Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502744288Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502755048Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502879269Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502905087Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389} {"msg":"de202393-c9de-4575-ada9-b27915d1c9ed active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502921546Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502930876Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"client":"0","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502940795Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"client":"1","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502953024Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"client":"2","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502963093Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.502976023Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38034)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.50301266Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"io task","client":"2","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.503023859Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36375)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.503113873Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"io task","client":"1","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"ds_connection connected from Ok(127.0.0.1:58303)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.503125822Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"io task","client":"0","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.503685384Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"client":"2","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.503725802Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"client":"0","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.503832254Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"client":"1","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.503991224Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504012582Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"[0]R flush_numbers[0..12]: [109, 110, 70, 107, 103, 100, 105, 113, 83, 94, 109, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504113655Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"downstairs","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"[0]R generation[0..12]: [62, 62, 42, 62, 60, 57, 60, 63, 52, 55, 62, 59]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504125224Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"downstairs","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504136744Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"downstairs","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"[1]R flush_numbers[0..12]: [109, 110, 70, 107, 103, 100, 105, 113, 83, 94, 109, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504150143Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"downstairs","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"[1]R generation[0..12]: [62, 62, 42, 62, 60, 57, 60, 63, 52, 55, 62, 59]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504160772Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"downstairs","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504171951Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"downstairs","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"[2]R flush_numbers[0..12]: [109, 110, 70, 107, 103, 100, 105, 113, 83, 94, 109, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.50418282Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"downstairs","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"[2]R generation[0..12]: [62, 62, 42, 62, 60, 57, 60, 63, 52, 55, 62, 59]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.50419372Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"downstairs","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504204849Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"downstairs","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504214618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"downstairs","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504224638Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"downstairs","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504234267Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"downstairs","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504243876Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"":"downstairs","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504254216Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504263405Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"client":"0","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504273644Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"client":"1","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504283844Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"client":"2","":"downstairs","session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504294253Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504304992Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"de202393-c9de-4575-ada9-b27915d1c9ed is now active with session: 2b395523-dce6-47bf-acaf-e587a16bd36d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504315381Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504324461Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"session_id":"2b395523-dce6-47bf-acaf-e587a16bd36d"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.50433384Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389} {"msg":"Activated sub_volume de202393-c9de-4575-ada9-b27915d1c9ed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.50434222Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504350649Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.504660078Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"task":"crutest"} 01/30 Write block 205 len 32768 data: 8 6 5 8 6 7 6 5 02/30 Write block 507 len 20480 data: 10 9 9 8 7 03/30 Read block 246 len 4096 04/30 Write block 406 len 32768 data: 6 6 6 4 4 6 6 7 05/30 Read block 44 len 8192 06/30 Read block 122 len 36864 07/30 Read block 250 len 12288 08/30 Write block 390 len 12288 data: 4 5 4 09/30 Read block 299 len 8192 10/30 Write block 160 len 8192 data: 5 4 11/30 Read block 424 len 16384 12/30 Write block 465 len 28672 data: 10 10 7 7 8 8 7 13/30 Read block 425 len 8192 14/30 Read block 451 len 32768 15/30 Read block 329 len 12288 16/30 Write block 135 len 20480 data: 4 5 6 6 7 17/30 Read block 587 len 12288 18/30 Write block 306 len 4096 data: 9 19/30 Read block 401 len 16384 20/30 Read block 543 len 20480 21/30 Read block 580 len 28672 22/30 Read block 548 len 12288 23/30 Write block 562 len 24576 data: 4 4 4 5 5 7 24/30 Write block 561 len 32768 data: 6 5 5 5 6 6 8 8 25/30 Write block 314 len 36864 data: 10 8 10 11 11 11 10 8 8 26/30 Read block 501 len 40960 27/30 Read block 513 len 16384 28/30 Read block 77 len 32768 29/30 Read block 224 len 40960 30/30 Read block 166 len 12288 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 5 Done Sent Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Write 8 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Read 4 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 30 1 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:06.513356946Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:06.513640607Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1389,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 109 109 109 F F F 1 020-039 62 62 62 110 110 110 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 60 60 60 103 103 103 F F F 5 100-119 57 57 57 100 100 100 F F F 6 120-139 60 60 60 105 105 105 T F T <--- 7 140-159 63 63 63 113 113 113 F F F 8 160-179 52 52 52 83 83 83 T F T <--- 9 180-199 55 55 55 94 94 94 F F F 10 200-219 62 62 62 109 109 109 T T T 11 220-239 59 59 59 102 102 102 F F F 12 240-259 60 60 60 104 104 104 F F F 13 260-279 58 58 58 101 101 101 F F F 14 280-299 53 53 53 85 85 85 F F F 15 300-319 62 62 62 109 109 109 T F T <--- 16 320-339 62 62 62 109 109 109 T F T <--- 17 340-359 62 62 62 109 109 109 F F F 18 360-379 62 62 62 109 109 109 F F F 19 380-399 61 61 61 106 106 106 T F T <--- 20 400-419 61 61 61 106 106 106 T F T <--- 21 420-439 61 61 61 106 106 106 F F F 22 440-459 63 63 63 113 113 113 F F F 23 460-479 61 61 61 106 106 106 T F T <--- 24 480-499 62 62 62 109 109 109 F F F 25 500-519 60 60 60 103 103 103 T F T <--- 26 520-539 59 59 59 102 102 102 F F F 27 540-559 59 59 59 102 102 102 F F F 28 560-579 57 57 57 100 100 100 T F T <--- 29 580-599 62 62 62 112 112 112 F F F Max gen: 63, Max flush: 113 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 109 109 109 F F F 1 020-039 62 62 62 110 110 110 F F F 2 040-059 42 42 42 70 70 70 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 60 60 60 103 103 103 F F F 5 100-119 57 57 57 100 100 100 F F F 6 120-139 64 64 64 114 114 114 F F F 7 140-159 63 63 63 113 113 113 F F F 8 160-179 64 64 64 114 114 114 F F F 9 180-199 55 55 55 94 94 94 F F F 10 200-219 64 64 64 114 114 114 F F F 11 220-239 59 59 59 102 102 102 F F F 12 240-259 60 60 60 104 104 104 F F F 13 260-279 58 58 58 101 101 101 F F F 14 280-299 53 53 53 85 85 85 F F F 15 300-319 64 64 64 114 114 114 F F F 16 320-339 64 64 64 114 114 114 F F F 17 340-359 62 62 62 109 109 109 F F F 18 360-379 62 62 62 109 109 109 F F F 19 380-399 64 64 64 114 114 114 F F F 20 400-419 64 64 64 114 114 114 F F F 21 420-439 61 61 61 106 106 106 F F F 22 440-459 63 63 63 113 113 113 F F F 23 460-479 64 64 64 114 114 114 F F F 24 480-499 62 62 62 109 109 109 F F F 25 500-519 64 64 64 114 114 114 F F F 26 520-539 59 59 59 102 102 102 F F F 27 540-559 59 59 59 102 102 102 F F F 28 560-579 64 64 64 114 114 114 F F F 29 580-599 62 62 62 112 112 112 F F F Max gen: 64, Max flush: 114 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.756394444Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.764482524Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.764763385Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.764793273Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.764802112Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.764813901Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397} {"msg":"Crucible 30893052-cacc-455a-8f6c-18ae97418857 has session id: 02b6caad-3cae-4b95-8e1a-0bb81a59fa71","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765190676Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"Upstairs opts: Upstairs UUID: 30893052-cacc-455a-8f6c-18ae97418857, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765220864Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"Crucible stats registered with UUID: 30893052-cacc-455a-8f6c-18ae97418857","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765234263Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:10.765244032Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765406911Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765437019Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765448028Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765606488Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765625016Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397} {"msg":"30893052-cacc-455a-8f6c-18ae97418857 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765663184Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765672823Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"client":"0","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765682862Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"client":"1","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765692832Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"client":"2","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765702721Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.76571328Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"ds_connection connected from Ok(127.0.0.1:57032)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765772196Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"io task","client":"2","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765783086Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"ds_connection connected from Ok(127.0.0.1:52736)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765793645Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"io task","client":"1","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"ds_connection connected from Ok(127.0.0.1:33874)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.765806264Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"io task","client":"0","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.76645234Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"client":"2","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766495487Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"client":"1","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766515156Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"client":"0","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.76675219Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766789057Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"[0]R flush_numbers[0..12]: [109, 110, 70, 107, 103, 100, 114, 113, 114, 94, 114, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766835264Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"downstairs","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"[0]R generation[0..12]: [62, 62, 42, 62, 60, 57, 64, 63, 64, 55, 64, 59]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766846463Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"downstairs","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766857692Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"downstairs","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"[1]R flush_numbers[0..12]: [109, 110, 70, 107, 103, 100, 114, 113, 114, 94, 114, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766868392Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"downstairs","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"[1]R generation[0..12]: [62, 62, 42, 62, 60, 57, 64, 63, 64, 55, 64, 59]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766878911Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"downstairs","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.76689012Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"downstairs","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"[2]R flush_numbers[0..12]: [109, 110, 70, 107, 103, 100, 114, 113, 114, 94, 114, 102]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766900909Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"downstairs","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"[2]R generation[0..12]: [62, 62, 42, 62, 60, 57, 64, 63, 64, 55, 64, 59]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766911449Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"downstairs","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766924478Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"downstairs","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"Max found gen is 65","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766934217Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"downstairs","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"Generation requested: 66 >= found:65","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766944147Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"downstairs","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"Next flush: 115","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766953796Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"downstairs","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766963435Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"":"downstairs","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766973455Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766982454Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"client":"0","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.766992423Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"client":"1","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.767002323Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"client":"2","":"downstairs","session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.767012622Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.767022141Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"30893052-cacc-455a-8f6c-18ae97418857 is now active with session: 02b6caad-3cae-4b95-8e1a-0bb81a59fa71","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.767032521Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.76704159Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"session_id":"02b6caad-3cae-4b95-8e1a-0bb81a59fa71"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.767050939Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397} {"msg":"Activated sub_volume 30893052-cacc-455a-8f6c-18ae97418857","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.767059239Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.767067758Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.767369618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"task":"crutest"} 01/30 Write block 459 len 12288 data: 6 5 4 02/30 Read block 277 len 12288 03/30 Write block 51 len 8192 data: 3 2 04/30 Read block 187 len 28672 05/30 Flush 06/30 Write block 485 len 20480 data: 6 7 9 7 7 07/30 Write block 246 len 4096 data: 8 08/30 Read block 133 len 16384 09/30 Read block 82 len 28672 10/30 Read block 182 len 40960 11/30 Read block 577 len 20480 12/30 Write block 227 len 24576 data: 7 8 5 6 7 8 13/30 Read block 181 len 12288 14/30 Read block 367 len 16384 15/30 Read block 59 len 20480 16/30 Read block 204 len 20480 17/30 Write block 178 len 8192 data: 3 3 18/30 Write block 489 len 32768 data: 8 7 5 5 6 4 3 4 19/30 Read block 598 len 8192 20/30 Write block 531 len 40960 data: 6 7 6 4 5 4 5 5 5 5 21/30 Write block 257 len 8192 data: 5 5 22/30 Read block 580 len 40960 23/30 Write block 262 len 12288 data: 5 5 5 24/30 Read block 243 len 24576 25/30 Write block 332 len 16384 data: 11 9 7 6 26/30 Flush 27/30 Write block 578 len 12288 data: 11 10 10 28/30 Write block 39 len 40960 data: 8 9 9 7 6 6 6 6 5 4 29/30 Read block 295 len 12288 30/30 Read block 235 len 32768 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Read 5 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Write 10 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 1 27 Done 30 4 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:10.77630231Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:10.776662505Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1397,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 109 109 109 F F F 1 020-039 62 62 62 110 110 110 T F T <--- 2 040-059 66 42 66 115 70 115 T T T <--- 3 060-079 62 62 62 107 107 107 F F F 4 080-099 60 60 60 103 103 103 F F F 5 100-119 57 57 57 100 100 100 F F F 6 120-139 64 64 64 114 114 114 F F F 7 140-159 63 63 63 113 113 113 F F F 8 160-179 66 64 66 116 114 116 F F F <--- 9 180-199 55 55 55 94 94 94 F F F 10 200-219 64 64 64 114 114 114 F F F 11 220-239 66 59 66 116 102 116 F F F <--- 12 240-259 66 60 66 116 104 116 F F F <--- 13 260-279 66 58 66 116 101 116 F F F <--- 14 280-299 53 53 53 85 85 85 F F F 15 300-319 64 64 64 114 114 114 F F F 16 320-339 66 64 66 116 114 116 F F F <--- 17 340-359 62 62 62 109 109 109 F F F 18 360-379 62 62 62 109 109 109 F F F 19 380-399 64 64 64 114 114 114 F F F 20 400-419 64 64 64 114 114 114 F F F 21 420-439 61 61 61 106 106 106 F F F 22 440-459 66 63 66 115 113 115 F T F <--- 23 460-479 66 64 66 115 114 115 F T F <--- 24 480-499 66 62 66 116 109 116 F F F <--- 25 500-519 64 64 64 114 114 114 F F F 26 520-539 66 59 66 116 102 116 F F F <--- 27 540-559 66 59 66 116 102 116 F F F <--- 28 560-579 64 64 64 114 114 114 T F T <--- 29 580-599 62 62 62 112 112 112 T F T <--- Max gen: 66, Max flush: 116 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 109 109 109 F F F 1 020-039 67 67 67 117 117 117 F F F 2 040-059 67 67 67 117 117 117 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 60 60 60 103 103 103 F F F 5 100-119 57 57 57 100 100 100 F F F 6 120-139 64 64 64 114 114 114 F F F 7 140-159 63 63 63 113 113 113 F F F 8 160-179 66 66 66 116 116 116 F F F 9 180-199 55 55 55 94 94 94 F F F 10 200-219 64 64 64 114 114 114 F F F 11 220-239 66 66 66 116 116 116 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 53 53 53 85 85 85 F F F 15 300-319 64 64 64 114 114 114 F F F 16 320-339 66 66 66 116 116 116 F F F 17 340-359 62 62 62 109 109 109 F F F 18 360-379 62 62 62 109 109 109 F F F 19 380-399 64 64 64 114 114 114 F F F 20 400-419 64 64 64 114 114 114 F F F 21 420-439 61 61 61 106 106 106 F F F 22 440-459 66 66 66 115 115 115 F F F 23 460-479 66 66 66 115 115 115 F F F 24 480-499 66 66 66 116 116 116 F F F 25 500-519 64 64 64 114 114 114 F F F 26 520-539 66 66 66 116 116 116 F F F 27 540-559 66 66 66 116 116 116 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 67 67 67 117 117 117 F F F Max gen: 67, Max flush: 117 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.022334984Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.030588692Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.030864584Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.030893282Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.030902531Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.03091446Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405} {"msg":"Crucible 287f875e-11d7-4cbb-ab6e-c8277a92788b has session id: d45f40d0-90a9-4c28-8d06-eac8632d6880","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031272046Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"Upstairs opts: Upstairs UUID: 287f875e-11d7-4cbb-ab6e-c8277a92788b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031300594Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"Crucible stats registered with UUID: 287f875e-11d7-4cbb-ab6e-c8277a92788b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031321362Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:15.031330982Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031476712Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.03150814Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031519759Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031618682Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031643251Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405} {"msg":"287f875e-11d7-4cbb-ab6e-c8277a92788b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031661229Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031670569Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"client":"0","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031680668Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"client":"1","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031690757Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"client":"2","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031700847Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031726165Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031737034Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"ds_connection connected from Ok(127.0.0.1:45909)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031767842Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"io task","client":"2","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"ds_connection connected from Ok(127.0.0.1:40321)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.031786621Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"io task","client":"1","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"ds_connection connected from Ok(127.0.0.1:40496)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.03179726Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"io task","client":"0","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.032510542Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"client":"0","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.032541209Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"client":"2","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.032553259Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"client":"1","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.032835999Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.032863438Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"[0]R flush_numbers[0..12]: [109, 117, 117, 107, 103, 100, 114, 113, 116, 94, 114, 116]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.032911884Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"downstairs","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"[0]R generation[0..12]: [62, 67, 67, 62, 60, 57, 64, 63, 66, 55, 64, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.032924113Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"downstairs","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.032935723Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"downstairs","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"[1]R flush_numbers[0..12]: [109, 117, 117, 107, 103, 100, 114, 113, 116, 94, 114, 116]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.032946772Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"downstairs","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"[1]R generation[0..12]: [62, 67, 67, 62, 60, 57, 64, 63, 66, 55, 64, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.032957481Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"downstairs","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.03296869Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"downstairs","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"[2]R flush_numbers[0..12]: [109, 117, 117, 107, 103, 100, 114, 113, 116, 94, 114, 116]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.03297999Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"downstairs","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"[2]R generation[0..12]: [62, 67, 67, 62, 60, 57, 64, 63, 66, 55, 64, 66]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.032990759Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"downstairs","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033002178Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"downstairs","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033012107Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"downstairs","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033022067Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"downstairs","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033031706Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"downstairs","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033041525Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"":"downstairs","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033051735Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033062584Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"client":"0","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033072743Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"client":"1","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033082673Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"client":"2","":"downstairs","session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033092802Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033102271Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"287f875e-11d7-4cbb-ab6e-c8277a92788b is now active with session: d45f40d0-90a9-4c28-8d06-eac8632d6880","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033112501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.03312208Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033129989Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"session_id":"d45f40d0-90a9-4c28-8d06-eac8632d6880"} {"msg":"Activated sub_volume 287f875e-11d7-4cbb-ab6e-c8277a92788b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033139519Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033147968Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.033494985Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"task":"crutest"} 01/30 Flush 02/30 Write block 333 len 8192 data: 10 8 03/30 Write block 345 len 8192 data: 6 6 04/30 Read block 36 len 20480 05/30 Flush 06/30 Flush 07/30 Write block 190 len 40960 data: 6 6 6 5 5 6 3 2 4 5 08/30 Read block 21 len 8192 09/30 Write block 542 len 28672 data: 5 5 6 6 5 5 7 10/30 Read block 344 len 36864 11/30 Read block 543 len 40960 12/30 Read block 227 len 40960 13/30 Read block 574 len 36864 14/30 Write block 281 len 8192 data: 4 5 15/30 Write block 220 len 40960 data: 5 6 6 5 8 5 7 8 9 6 16/30 Read block 499 len 28672 17/30 Flush 18/30 Write block 201 len 4096 data: 7 19/30 Write block 383 len 16384 data: 7 4 4 3 20/30 Write block 199 len 8192 data: 6 7 21/30 Write block 349 len 4096 data: 6 22/30 Read block 548 len 8192 23/30 Write block 38 len 8192 data: 6 9 24/30 Read block 277 len 8192 25/30 Write block 553 len 20480 data: 5 6 8 7 6 26/30 Read block 134 len 36864 27/30 Write block 145 len 36864 data: 6 7 8 7 8 8 8 7 4 28/30 Write block 342 len 16384 data: 6 6 7 7 29/30 Read block 434 len 24576 30/30 Read block 195 len 20480 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 2 Done Done Done false 1002 Acked Write 2 Done Sent Done false 1003 Acked Read 5 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 9 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 29 1 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1000)) Some(JobId(1016)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:15.042109138Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:15.042400849Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1405,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 109 109 109 F F F 1 020-039 67 67 67 117 117 117 T F T <--- 2 040-059 67 67 67 117 117 117 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 60 60 60 103 103 103 F F F 5 100-119 57 57 57 100 100 100 F F F 6 120-139 64 64 64 114 114 114 F F F 7 140-159 63 63 63 113 113 113 T F T <--- 8 160-179 66 66 66 116 116 116 F F F 9 180-199 68 55 68 121 94 121 T F T <--- 10 200-219 64 64 64 114 114 114 T F T <--- 11 220-239 68 66 68 121 116 121 F F F <--- 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 68 53 68 121 85 121 F F F <--- 15 300-319 64 64 64 114 114 114 F F F 16 320-339 68 66 68 119 116 119 F T F <--- 17 340-359 68 62 68 119 109 119 T F T <--- 18 360-379 62 62 62 109 109 109 F F F 19 380-399 64 64 64 114 114 114 T F T <--- 20 400-419 64 64 64 114 114 114 F F F 21 420-439 61 61 61 106 106 106 F F F 22 440-459 66 66 66 115 115 115 F F F 23 460-479 66 66 66 115 115 115 F F F 24 480-499 66 66 66 116 116 116 F F F 25 500-519 64 64 64 114 114 114 F F F 26 520-539 66 66 66 116 116 116 F F F 27 540-559 68 66 68 121 116 121 T F T <--- 28 560-579 67 67 67 117 117 117 F F F 29 580-599 67 67 67 117 117 117 F F F Max gen: 68, Max flush: 121 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 109 109 109 F F F 1 020-039 69 69 69 122 122 122 F F F 2 040-059 67 67 67 117 117 117 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 60 60 60 103 103 103 F F F 5 100-119 57 57 57 100 100 100 F F F 6 120-139 64 64 64 114 114 114 F F F 7 140-159 69 69 69 122 122 122 F F F 8 160-179 66 66 66 116 116 116 F F F 9 180-199 69 69 69 122 122 122 F F F 10 200-219 69 69 69 122 122 122 F F F 11 220-239 68 68 68 121 121 121 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 68 68 68 121 121 121 F F F 15 300-319 64 64 64 114 114 114 F F F 16 320-339 68 68 68 119 119 119 F F F 17 340-359 69 69 69 122 122 122 F F F 18 360-379 62 62 62 109 109 109 F F F 19 380-399 69 69 69 122 122 122 F F F 20 400-419 64 64 64 114 114 114 F F F 21 420-439 61 61 61 106 106 106 F F F 22 440-459 66 66 66 115 115 115 F F F 23 460-479 66 66 66 115 115 115 F F F 24 480-499 66 66 66 116 116 116 F F F 25 500-519 64 64 64 114 114 114 F F F 26 520-539 66 66 66 116 116 116 F F F 27 540-559 69 69 69 122 122 122 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 67 67 67 117 117 117 F F F Max gen: 69, Max flush: 122 Begin loop 35 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.280919284Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.289276325Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.289589784Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.289627272Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.289636841Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.28965142Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413} {"msg":"Crucible 0e488ab9-7e61-4376-a33d-bf6f147ea3b1 has session id: a3d4bcad-48e7-4d9c-a656-c026cfda4eed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.289925851Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"Upstairs opts: Upstairs UUID: 0e488ab9-7e61-4376-a33d-bf6f147ea3b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.28994912Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"Crucible stats registered with UUID: 0e488ab9-7e61-4376-a33d-bf6f147ea3b1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.289959369Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:19.289972828Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.290148286Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.290180124Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.290191383Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.290292236Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.290309965Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413} {"msg":"0e488ab9-7e61-4376-a33d-bf6f147ea3b1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.290350132Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.290359752Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"client":"0","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.290369831Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"client":"1","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.29038Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"client":"2","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.29039011Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.290402859Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.290413258Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"ds_connection connected from Ok(127.0.0.1:34903)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.290437157Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"io task","client":"2","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"ds_connection connected from Ok(127.0.0.1:52726)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.290478944Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"io task","client":"1","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"ds_connection connected from Ok(127.0.0.1:56597)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.290616954Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"io task","client":"0","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291167177Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"client":"1","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291208174Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"client":"0","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291233432Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"client":"2","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291472696Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291523083Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"[0]R flush_numbers[0..12]: [109, 122, 117, 107, 103, 100, 114, 122, 116, 122, 122, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291554161Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"downstairs","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"[0]R generation[0..12]: [62, 69, 67, 62, 60, 57, 64, 69, 66, 69, 69, 68]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.29156542Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"downstairs","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291576909Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"downstairs","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"[1]R flush_numbers[0..12]: [109, 122, 117, 107, 103, 100, 114, 122, 116, 122, 122, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291588028Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"downstairs","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"[1]R generation[0..12]: [62, 69, 67, 62, 60, 57, 64, 69, 66, 69, 69, 68]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291598697Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"downstairs","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291612677Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"downstairs","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"[2]R flush_numbers[0..12]: [109, 122, 117, 107, 103, 100, 114, 122, 116, 122, 122, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291623786Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"downstairs","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"[2]R generation[0..12]: [62, 69, 67, 62, 60, 57, 64, 69, 66, 69, 69, 68]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291634395Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"downstairs","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291645594Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"downstairs","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291655294Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"downstairs","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291665233Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"downstairs","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291674832Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"downstairs","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291684402Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"":"downstairs","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291694371Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.29170341Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"client":"0","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.29171343Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"client":"1","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291723319Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"client":"2","":"downstairs","session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291733308Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291742718Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"0e488ab9-7e61-4376-a33d-bf6f147ea3b1 is now active with session: a3d4bcad-48e7-4d9c-a656-c026cfda4eed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291752867Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291763526Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291771526Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"session_id":"a3d4bcad-48e7-4d9c-a656-c026cfda4eed"} {"msg":"Activated sub_volume 0e488ab9-7e61-4376-a33d-bf6f147ea3b1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291780905Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.291789295Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.292075375Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"task":"crutest"} 01/30 Write block 339 len 28672 data: 7 6 6 7 7 8 8 02/30 Read block 469 len 32768 03/30 Write block 458 len 36864 data: 8 7 6 5 4 7 8 11 11 04/30 Read block 478 len 20480 05/30 Read block 532 len 20480 06/30 Write block 114 len 24576 data: 6 5 5 4 4 4 07/30 Read block 323 len 24576 08/30 Read block 284 len 12288 09/30 Read block 232 len 32768 10/30 Write block 435 len 40960 data: 8 7 6 6 5 6 7 6 5 3 11/30 Read block 217 len 8192 12/30 Read block 59 len 36864 13/30 Read block 133 len 32768 14/30 Read block 451 len 40960 15/30 Read block 409 len 16384 16/30 Write block 372 len 20480 data: 7 7 7 9 8 17/30 Flush 18/30 Write block 213 len 36864 data: 4 4 6 6 4 5 5 6 7 19/30 Read block 499 len 24576 20/30 Read block 568 len 8192 21/30 Write block 555 len 4096 data: 9 22/30 Flush 23/30 Write block 205 len 36864 data: 9 7 6 9 7 8 7 6 5 24/30 Read block 242 len 16384 25/30 Flush 26/30 Flush 27/30 Write block 142 len 28672 data: 8 8 8 7 8 9 8 28/30 Read block 416 len 8192 29/30 Write block 312 len 20480 data: 11 10 11 9 11 30/30 Write block 524 len 24576 data: 4 6 6 6 5 6 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 5 Done Done Sent false 1005 Acked Write 6 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 26 30 Done 28 28 4 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:19.30097239Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:19.301284278Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1413,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 109 109 109 F F F 1 020-039 69 69 69 122 122 122 F F F 2 040-059 67 67 67 117 117 117 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 60 60 60 103 103 103 F F F 5 100-119 70 70 57 123 123 100 F F F <--- 6 120-139 64 64 64 114 114 114 F F F 7 140-159 69 69 69 122 122 122 T T F <--- 8 160-179 66 66 66 116 116 116 F F F 9 180-199 69 69 69 122 122 122 F F F 10 200-219 70 70 69 125 125 122 F F F <--- 11 220-239 70 70 68 124 124 121 F F F <--- 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 68 68 68 121 121 121 F F F 15 300-319 64 64 64 114 114 114 T T F <--- 16 320-339 70 70 68 123 123 119 F F T <--- 17 340-359 70 70 69 123 123 122 F F T <--- 18 360-379 70 70 62 123 123 109 F F F <--- 19 380-399 69 69 69 122 122 122 F F F 20 400-419 64 64 64 114 114 114 F F F 21 420-439 70 70 61 123 123 106 F F F <--- 22 440-459 70 70 66 123 123 115 F F T <--- 23 460-479 70 70 66 123 123 115 F F T <--- 24 480-499 66 66 66 116 116 116 F F F 25 500-519 64 64 64 114 114 114 F F F 26 520-539 66 66 66 116 116 116 T T F <--- 27 540-559 70 70 69 124 124 122 F F F <--- 28 560-579 67 67 67 117 117 117 F F F 29 580-599 67 67 67 117 117 117 F F F Max gen: 70, Max flush: 125 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 109 109 109 F F F 1 020-039 69 69 69 122 122 122 F F F 2 040-059 67 67 67 117 117 117 F F F 3 060-079 62 62 62 107 107 107 F F F 4 080-099 60 60 60 103 103 103 F F F 5 100-119 70 70 70 123 123 123 F F F 6 120-139 64 64 64 114 114 114 F F F 7 140-159 71 71 71 126 126 126 F F F 8 160-179 66 66 66 116 116 116 F F F 9 180-199 69 69 69 122 122 122 F F F 10 200-219 70 70 70 125 125 125 F F F 11 220-239 70 70 70 124 124 124 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 68 68 68 121 121 121 F F F 15 300-319 71 71 71 126 126 126 F F F 16 320-339 70 70 70 123 123 123 F F F 17 340-359 70 70 70 123 123 123 F F F 18 360-379 70 70 70 123 123 123 F F F 19 380-399 69 69 69 122 122 122 F F F 20 400-419 64 64 64 114 114 114 F F F 21 420-439 70 70 70 123 123 123 F F F 22 440-459 70 70 70 123 123 123 F F F 23 460-479 70 70 70 123 123 123 F F F 24 480-499 66 66 66 116 116 116 F F F 25 500-519 64 64 64 114 114 114 F F F 26 520-539 71 71 71 126 126 126 F F F 27 540-559 70 70 70 124 124 124 F F F 28 560-579 67 67 67 117 117 117 F F F 29 580-599 67 67 67 117 117 117 F F F Max gen: 71, Max flush: 126 Begin loop 36 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.544065164Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:23.551118354Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.55117587Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.558199032Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.558442596Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.558470434Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.558479733Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.558491452Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421} {"msg":"Crucible 7f3bce53-d74e-4b2f-b804-b069786f43de has session id: 7f2198da-6aad-4902-949c-d23ec3b96d6c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.558876636Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"Upstairs opts: Upstairs UUID: 7f3bce53-d74e-4b2f-b804-b069786f43de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.558898705Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"Crucible stats registered with UUID: 7f3bce53-d74e-4b2f-b804-b069786f43de","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.558911244Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:23.558923203Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.559123079Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"io task","client":"0","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.559155587Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"io task","client":"2","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.559166806Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"io task","client":"1","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.559301027Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.559356834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421} {"msg":"7f3bce53-d74e-4b2f-b804-b069786f43de active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.559392021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.55940278Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"client":"0","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.55941399Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"client":"1","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.559424209Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"client":"2","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.559435278Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"io task","client":"0","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.559445927Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"io task","client":"2","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.559460076Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"io task","client":"1","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62005)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.559512463Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"io task","client":"2","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55250)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.559532782Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"io task","client":"1","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55422)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.559545701Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"io task","client":"0","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560229714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"client":"1","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560308989Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"client":"2","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.56044284Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"client":"0","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560649226Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560689393Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"[0]R flush_numbers[0..12]: [109, 122, 117, 107, 103, 123, 114, 126, 116, 122, 125, 124]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560838593Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"downstairs","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"[0]R generation[0..12]: [62, 69, 67, 62, 60, 70, 64, 71, 66, 69, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560850202Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"downstairs","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560861781Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"downstairs","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"[1]R flush_numbers[0..12]: [109, 122, 117, 107, 103, 123, 114, 126, 116, 122, 125, 124]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.56087295Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"downstairs","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"[1]R generation[0..12]: [62, 69, 67, 62, 60, 70, 64, 71, 66, 69, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.56088382Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"downstairs","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560895209Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"downstairs","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"[2]R flush_numbers[0..12]: [109, 122, 117, 107, 103, 123, 114, 126, 116, 122, 125, 124]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560906178Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"downstairs","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"[2]R generation[0..12]: [62, 69, 67, 62, 60, 70, 64, 71, 66, 69, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560917157Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"downstairs","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560930426Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"downstairs","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560940276Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"downstairs","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560950535Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"downstairs","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560960214Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"downstairs","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560970094Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"":"downstairs","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560980523Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.560989882Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"client":"0","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.561000082Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"client":"1","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.561010391Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"client":"2","":"downstairs","session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.56102084Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.56103038Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"7f3bce53-d74e-4b2f-b804-b069786f43de is now active with session: 7f2198da-6aad-4902-949c-d23ec3b96d6c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.561040879Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.561050288Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.561058548Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"session_id":"7f2198da-6aad-4902-949c-d23ec3b96d6c"} {"msg":"Activated sub_volume 7f3bce53-d74e-4b2f-b804-b069786f43de","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.561068137Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.561076477Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.561411104Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"task":"crutest"} 01/30 Write block 187 len 8192 data: 5 7 02/30 Read block 476 len 12288 03/30 Read block 140 len 40960 04/30 Read block 79 len 20480 05/30 Read block 219 len 40960 06/30 Flush 07/30 Write block 298 len 4096 data: 7 08/30 Read block 541 len 24576 09/30 Read block 243 len 8192 10/30 Write block 297 len 12288 data: 6 8 6 11/30 Read block 491 len 28672 12/30 Read block 317 len 32768 13/30 Read block 355 len 40960 14/30 Write block 570 len 32768 data: 9 7 8 9 8 12 11 11 15/30 Write block 399 len 8192 data: 9 7 16/30 Read block 138 len 36864 17/30 Write block 549 len 40960 data: 7 6 5 5 6 7 10 8 7 6 18/30 Flush 19/30 Write block 64 len 8192 data: 7 6 20/30 Write block 517 len 4096 data: 3 21/30 Write block 491 len 36864 data: 6 6 7 5 4 5 4 2 3 22/30 Read block 151 len 24576 23/30 Read block 246 len 16384 24/30 Write block 394 len 40960 data: 8 9 8 9 9 10 8 7 7 6 25/30 Read block 268 len 8192 26/30 Read block 169 len 36864 27/30 Read block 393 len 12288 28/30 Read block 263 len 8192 29/30 Read block 249 len 28672 30/30 Write block 13 len 8192 data: 7 5 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 1 Done Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Read 2 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 2 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 1 1 25 Done 1 23 23 47 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1005)) Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:23.570531823Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:23.570819044Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1421,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 109 109 109 F T T <--- 1 020-039 69 69 69 122 122 122 F F F 2 040-059 67 67 67 117 117 117 F F F 3 060-079 62 62 62 107 107 107 F T T <--- 4 080-099 60 60 60 103 103 103 F F F 5 100-119 70 70 70 123 123 123 F F F 6 120-139 64 64 64 114 114 114 F F F 7 140-159 71 71 71 126 126 126 F F F 8 160-179 66 66 66 116 116 116 F F F 9 180-199 72 72 72 127 127 127 F F F 10 200-219 70 70 70 125 125 125 F F F 11 220-239 70 70 70 124 124 124 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 68 72 72 121 128 128 T F F <--- 15 300-319 71 71 71 126 126 126 F F F 16 320-339 70 70 70 123 123 123 F F F 17 340-359 70 70 70 123 123 123 F F F 18 360-379 70 70 70 123 123 123 F F F 19 380-399 69 72 72 122 128 128 F T T <--- 20 400-419 64 72 72 114 128 128 F T T <--- 21 420-439 70 70 70 123 123 123 F F F 22 440-459 70 70 70 123 123 123 F F F 23 460-479 70 70 70 123 123 123 F F F 24 480-499 66 66 66 116 116 116 F T T <--- 25 500-519 64 64 64 114 114 114 F T T <--- 26 520-539 71 71 71 126 126 126 F F F 27 540-559 70 72 72 124 128 128 F F F <--- 28 560-579 67 72 72 117 128 128 F F F <--- 29 580-599 67 67 67 117 117 117 F F F Max gen: 72, Max flush: 128 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 73 73 73 129 129 129 F F F 1 020-039 69 69 69 122 122 122 F F F 2 040-059 67 67 67 117 117 117 F F F 3 060-079 73 73 73 129 129 129 F F F 4 080-099 60 60 60 103 103 103 F F F 5 100-119 70 70 70 123 123 123 F F F 6 120-139 64 64 64 114 114 114 F F F 7 140-159 71 71 71 126 126 126 F F F 8 160-179 66 66 66 116 116 116 F F F 9 180-199 72 72 72 127 127 127 F F F 10 200-219 70 70 70 125 125 125 F F F 11 220-239 70 70 70 124 124 124 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 71 71 71 126 126 126 F F F 16 320-339 70 70 70 123 123 123 F F F 17 340-359 70 70 70 123 123 123 F F F 18 360-379 70 70 70 123 123 123 F F F 19 380-399 73 73 73 129 129 129 F F F 20 400-419 73 73 73 129 129 129 F F F 21 420-439 70 70 70 123 123 123 F F F 22 440-459 70 70 70 123 123 123 F F F 23 460-479 70 70 70 123 123 123 F F F 24 480-499 73 73 73 129 129 129 F F F 25 500-519 73 73 73 129 129 129 F F F 26 520-539 71 71 71 126 126 126 F F F 27 540-559 72 72 72 128 128 128 F F F 28 560-579 72 72 72 128 128 128 F F F 29 580-599 67 67 67 117 117 117 F F F Max gen: 73, Max flush: 129 Begin loop 37 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.817426689Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.825922471Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.826241759Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.826268177Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.826276966Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.826288806Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429} {"msg":"Crucible f10221f0-42bb-4859-8b13-d1b3d05a36fc has session id: 833007bb-a928-495a-a2b9-ceed833ee09b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.826579076Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"Upstairs opts: Upstairs UUID: f10221f0-42bb-4859-8b13-d1b3d05a36fc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.826609274Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"Crucible stats registered with UUID: f10221f0-42bb-4859-8b13-d1b3d05a36fc","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.826620313Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:27.826631702Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.82681928Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.826850637Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.826862147Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.826980649Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.826996088Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429} {"msg":"f10221f0-42bb-4859-8b13-d1b3d05a36fc active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.827036635Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.827046484Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"client":"0","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.827056563Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"client":"1","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.827066583Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"client":"2","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.827076552Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.827086941Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.827097051Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41180)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.827142058Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"io task","client":"0","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50077)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.827168296Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"io task","client":"2","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45327)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.827265909Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"io task","client":"1","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.827749616Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"client":"2","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.827816812Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"client":"0","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.827828671Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"client":"1","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828058775Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828102472Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"[0]R flush_numbers[0..12]: [129, 122, 117, 129, 103, 123, 114, 126, 116, 127, 125, 124]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828178117Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"downstairs","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"[0]R generation[0..12]: [73, 69, 67, 73, 60, 70, 64, 71, 66, 72, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828192046Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"downstairs","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828203375Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"downstairs","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"[1]R flush_numbers[0..12]: [129, 122, 117, 129, 103, 123, 114, 126, 116, 127, 125, 124]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828214335Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"downstairs","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"[1]R generation[0..12]: [73, 69, 67, 73, 60, 70, 64, 71, 66, 72, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828225074Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"downstairs","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828236373Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"downstairs","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"[2]R flush_numbers[0..12]: [129, 122, 117, 129, 103, 123, 114, 126, 116, 127, 125, 124]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828247392Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"downstairs","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"[2]R generation[0..12]: [73, 69, 67, 73, 60, 70, 64, 71, 66, 72, 70, 70]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828258072Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"downstairs","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828269291Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"downstairs","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.82827894Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"downstairs","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.82828917Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"downstairs","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828298759Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"downstairs","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828308528Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"":"downstairs","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828318808Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828327877Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"client":"0","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828339096Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"client":"1","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828349265Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"client":"2","":"downstairs","session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828359605Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828369164Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"f10221f0-42bb-4859-8b13-d1b3d05a36fc is now active with session: 833007bb-a928-495a-a2b9-ceed833ee09b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828379243Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828388623Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828396562Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"session_id":"833007bb-a928-495a-a2b9-ceed833ee09b"} {"msg":"Activated sub_volume f10221f0-42bb-4859-8b13-d1b3d05a36fc","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828405982Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.828414421Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.82872685Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"task":"crutest"} 01/30 Write block 7 len 28672 data: 8 6 6 7 6 6 8 02/30 Flush 03/30 Write block 138 len 36864 data: 7 8 7 8 9 9 9 8 9 04/30 Read block 219 len 40960 05/30 Write block 483 len 12288 data: 5 5 7 06/30 Read block 242 len 32768 07/30 Read block 558 len 8192 08/30 Read block 537 len 16384 09/30 Write block 231 len 4096 data: 8 10/30 Read block 315 len 16384 11/30 Read block 194 len 4096 12/30 Write block 81 len 20480 data: 6 4 3 3 3 13/30 Flush 14/30 Write block 448 len 16384 data: 6 6 6 3 15/30 Write block 125 len 8192 data: 8 8 16/30 Read block 3 len 8192 17/30 Read block 193 len 12288 18/30 Write block 232 len 8192 data: 9 6 19/30 Read block 155 len 32768 20/30 Read block 244 len 4096 21/30 Read block 294 len 12288 22/30 Read block 518 len 12288 23/30 Write block 228 len 4096 data: 10 24/30 Write block 19 len 8192 data: 7 7 25/30 Write block 583 len 16384 data: 6 5 7 6 26/30 Read block 442 len 16384 27/30 Read block 524 len 24576 28/30 Read block 454 len 40960 29/30 Read block 96 len 40960 30/30 Write block 128 len 32768 data: 6 5 5 5 6 4 5 5 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 9 Done Done Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 8 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 4 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 27 1 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1001)) Some(JobId(1012)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:27.83680583Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:27.837091341Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1429,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 74 74 74 130 130 130 T F T <--- 1 020-039 69 69 69 122 122 122 T F T <--- 2 040-059 67 67 67 117 117 117 F F F 3 060-079 73 73 73 129 129 129 F F F 4 080-099 74 60 74 131 103 131 F F F <--- 5 100-119 70 70 70 123 123 123 F F F 6 120-139 74 64 74 131 114 131 T T T <--- 7 140-159 74 71 74 131 126 131 F T F <--- 8 160-179 66 66 66 116 116 116 F F F 9 180-199 72 72 72 127 127 127 F F F 10 200-219 70 70 70 125 125 125 F F F 11 220-239 74 70 74 131 124 131 T F T <--- 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 71 71 71 126 126 126 F F F 16 320-339 70 70 70 123 123 123 F F F 17 340-359 70 70 70 123 123 123 F F F 18 360-379 70 70 70 123 123 123 F F F 19 380-399 73 73 73 129 129 129 F F F 20 400-419 73 73 73 129 129 129 F F F 21 420-439 70 70 70 123 123 123 F F F 22 440-459 70 70 70 123 123 123 T F T <--- 23 460-479 70 70 70 123 123 123 F F F 24 480-499 74 73 74 131 129 131 F F F <--- 25 500-519 73 73 73 129 129 129 F F F 26 520-539 71 71 71 126 126 126 F F F 27 540-559 72 72 72 128 128 128 F F F 28 560-579 72 72 72 128 128 128 F F F 29 580-599 67 67 67 117 117 117 T F T <--- Max gen: 74, Max flush: 131 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 132 132 132 F F F 1 020-039 75 75 75 132 132 132 F F F 2 040-059 67 67 67 117 117 117 F F F 3 060-079 73 73 73 129 129 129 F F F 4 080-099 74 74 74 131 131 131 F F F 5 100-119 70 70 70 123 123 123 F F F 6 120-139 75 75 75 132 132 132 F F F 7 140-159 74 74 74 131 131 131 F F F 8 160-179 66 66 66 116 116 116 F F F 9 180-199 72 72 72 127 127 127 F F F 10 200-219 70 70 70 125 125 125 F F F 11 220-239 75 75 75 132 132 132 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 71 71 71 126 126 126 F F F 16 320-339 70 70 70 123 123 123 F F F 17 340-359 70 70 70 123 123 123 F F F 18 360-379 70 70 70 123 123 123 F F F 19 380-399 73 73 73 129 129 129 F F F 20 400-419 73 73 73 129 129 129 F F F 21 420-439 70 70 70 123 123 123 F F F 22 440-459 75 75 75 132 132 132 F F F 23 460-479 70 70 70 123 123 123 F F F 24 480-499 74 74 74 131 131 131 F F F 25 500-519 73 73 73 129 129 129 F F F 26 520-539 71 71 71 126 126 126 F F F 27 540-559 72 72 72 128 128 128 F F F 28 560-579 72 72 72 128 128 128 F F F 29 580-599 75 75 75 132 132 132 F F F Max gen: 75, Max flush: 132 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.072728362Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:32.079988508Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.080049354Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.08702022Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.087268303Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.087296921Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.08730587Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.087320699Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437} {"msg":"Crucible e547ec1a-d7c7-4b02-8e68-ae97a4cc664d has session id: b7c22a83-6aa0-4500-9514-2788ea05d163","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.087656996Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"Upstairs opts: Upstairs UUID: e547ec1a-d7c7-4b02-8e68-ae97a4cc664d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.087688874Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"Crucible stats registered with UUID: e547ec1a-d7c7-4b02-8e68-ae97a4cc664d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.087700443Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:32.087710203Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.087867762Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"io task","client":"1","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.087887581Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"io task","client":"0","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.08789825Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"io task","client":"2","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088009082Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088024431Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437} {"msg":"e547ec1a-d7c7-4b02-8e68-ae97a4cc664d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.08804922Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088059029Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"client":"0","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088069268Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"client":"1","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088079298Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"client":"2","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088089157Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"io task","client":"2","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088103496Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"io task","client":"1","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088113995Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"io task","client":"0","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"ds_connection connected from Ok(127.0.0.1:36656)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088154233Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"io task","client":"2","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"ds_connection connected from Ok(127.0.0.1:44311)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088165742Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"io task","client":"1","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"ds_connection connected from Ok(127.0.0.1:63692)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088176611Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"io task","client":"0","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088817657Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"client":"2","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088843876Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"client":"1","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.088890163Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"client":"0","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089137516Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089200641Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"[0]R flush_numbers[0..12]: [132, 132, 117, 129, 131, 123, 132, 131, 116, 127, 125, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089262427Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"downstairs","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"[0]R generation[0..12]: [75, 75, 67, 73, 74, 70, 75, 74, 66, 72, 70, 75]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089274616Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"downstairs","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089286136Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"downstairs","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"[1]R flush_numbers[0..12]: [132, 132, 117, 129, 131, 123, 132, 131, 116, 127, 125, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089297235Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"downstairs","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"[1]R generation[0..12]: [75, 75, 67, 73, 74, 70, 75, 74, 66, 72, 70, 75]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089310104Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"downstairs","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089321613Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"downstairs","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"[2]R flush_numbers[0..12]: [132, 132, 117, 129, 131, 123, 132, 131, 116, 127, 125, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089332812Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"downstairs","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"[2]R generation[0..12]: [75, 75, 67, 73, 74, 70, 75, 74, 66, 72, 70, 75]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089343552Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"downstairs","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089354821Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"downstairs","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.0893647Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"downstairs","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.08937524Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"downstairs","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"Next flush: 133","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089385419Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"downstairs","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089395388Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"":"downstairs","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089405687Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089414967Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"client":"0","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089425056Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"client":"1","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089435005Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"client":"2","":"downstairs","session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089445165Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089454714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"e547ec1a-d7c7-4b02-8e68-ae97a4cc664d is now active with session: b7c22a83-6aa0-4500-9514-2788ea05d163","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089465083Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089476213Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089484222Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"session_id":"b7c22a83-6aa0-4500-9514-2788ea05d163"} {"msg":"Activated sub_volume e547ec1a-d7c7-4b02-8e68-ae97a4cc664d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089493641Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.089502111Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.0898091Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"task":"crutest"} 01/30 Flush 02/30 Write block 132 len 32768 data: 7 5 6 6 6 7 8 9 03/30 Write block 191 len 40960 data: 7 7 6 6 7 4 3 5 7 8 04/30 Write block 206 len 36864 data: 8 7 10 8 9 8 7 6 5 05/30 Write block 411 len 28672 data: 7 7 8 8 10 10 8 06/30 Flush 07/30 Write block 85 len 28672 data: 4 4 4 4 3 3 4 08/30 Read block 575 len 28672 09/30 Read block 115 len 12288 10/30 Write block 520 len 28672 data: 5 5 6 5 5 7 7 11/30 Write block 587 len 4096 data: 7 12/30 Read block 155 len 16384 13/30 Flush 14/30 Write block 354 len 40960 data: 6 7 8 8 9 10 9 12 9 8 15/30 Flush 16/30 Write block 150 len 24576 data: 9 9 8 5 5 4 17/30 Read block 85 len 32768 18/30 Flush 19/30 Write block 114 len 8192 data: 7 6 20/30 Write block 310 len 32768 data: 8 10 12 11 12 10 12 12 21/30 Flush 22/30 Write block 134 len 24576 data: 7 7 7 8 9 10 23/30 Read block 215 len 20480 24/30 Write block 497 len 16384 data: 5 3 4 4 25/30 Write block 31 len 28672 data: 10 7 10 8 7 7 5 26/30 Read block 58 len 8192 27/30 Write block 457 len 32768 data: 8 9 8 7 6 5 8 9 28/30 Read block 244 len 12288 29/30 Read block 256 len 4096 30/30 Read block 81 len 40960 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 8 Sent Done Done false 1002 Acked Write 10 Sent Done Done false 1003 Acked Write 9 Sent Done Done false 1004 Acked Write 7 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Read 3 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Read 1 Sent Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 0 0 29 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:32.099338762Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:32.099631082Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1437,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 132 132 132 F F F 1 020-039 75 75 75 132 132 132 F T T <--- 2 040-059 67 67 67 117 117 117 F F F 3 060-079 73 73 73 129 129 129 F F F 4 080-099 74 76 76 131 135 135 F F F <--- 5 100-119 70 76 76 123 138 138 F F F <--- 6 120-139 75 76 76 132 134 134 F T T <--- 7 140-159 74 76 76 131 137 137 F F F <--- 8 160-179 66 66 66 116 116 116 F F F 9 180-199 72 76 76 127 134 134 F F F <--- 10 200-219 70 76 76 125 134 134 F F F <--- 11 220-239 75 75 75 132 132 132 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 71 76 76 126 138 138 F F F <--- 16 320-339 70 70 70 123 123 123 F F F 17 340-359 70 76 76 123 136 136 F F F <--- 18 360-379 70 76 76 123 136 136 F F F <--- 19 380-399 73 73 73 129 129 129 F F F 20 400-419 73 76 76 129 134 134 F F F <--- 21 420-439 70 70 70 123 123 123 F F F 22 440-459 75 75 75 132 132 132 F T T <--- 23 460-479 70 70 70 123 123 123 F T T <--- 24 480-499 74 74 74 131 131 131 F T T <--- 25 500-519 73 73 73 129 129 129 F T T <--- 26 520-539 71 76 76 126 135 135 F F F <--- 27 540-559 72 72 72 128 128 128 F F F 28 560-579 72 72 72 128 128 128 F F F 29 580-599 75 76 76 132 135 135 F F F <--- Max gen: 76, Max flush: 138 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 132 132 132 F F F 1 020-039 77 77 77 139 139 139 F F F 2 040-059 67 67 67 117 117 117 F F F 3 060-079 73 73 73 129 129 129 F F F 4 080-099 76 76 76 135 135 135 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 77 77 77 139 139 139 F F F 7 140-159 76 76 76 137 137 137 F F F 8 160-179 66 66 66 116 116 116 F F F 9 180-199 76 76 76 134 134 134 F F F 10 200-219 76 76 76 134 134 134 F F F 11 220-239 75 75 75 132 132 132 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 76 76 76 138 138 138 F F F 16 320-339 70 70 70 123 123 123 F F F 17 340-359 76 76 76 136 136 136 F F F 18 360-379 76 76 76 136 136 136 F F F 19 380-399 73 73 73 129 129 129 F F F 20 400-419 76 76 76 134 134 134 F F F 21 420-439 70 70 70 123 123 123 F F F 22 440-459 77 77 77 139 139 139 F F F 23 460-479 77 77 77 139 139 139 F F F 24 480-499 77 77 77 139 139 139 F F F 25 500-519 77 77 77 139 139 139 F F F 26 520-539 76 76 76 135 135 135 F F F 27 540-559 72 72 72 128 128 128 F F F 28 560-579 72 72 72 128 128 128 F F F 29 580-599 76 76 76 135 135 135 F F F Max gen: 77, Max flush: 139 Begin loop 39 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.358770584Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:36.366003032Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.366094236Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.373072941Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.373349472Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.373374091Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.37338301Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.373395319Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445} {"msg":"Crucible d4f002cf-e725-4afa-9dd5-b946cfebf9ea has session id: 5a0b344e-aeb4-47bd-878f-1ed5a04c1f03","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.373754445Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"Upstairs opts: Upstairs UUID: d4f002cf-e725-4afa-9dd5-b946cfebf9ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.373785353Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"Crucible stats registered with UUID: d4f002cf-e725-4afa-9dd5-b946cfebf9ea","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.373798912Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:36.373808691Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.37397201Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374014237Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374026066Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374129129Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374154188Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445} {"msg":"d4f002cf-e725-4afa-9dd5-b946cfebf9ea active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374172066Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374181266Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"client":"0","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374191455Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"client":"1","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374201364Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"client":"2","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374211164Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374221563Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374231952Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"ds_connection connected from Ok(127.0.0.1:57221)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374304847Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"io task","client":"1","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"ds_connection connected from Ok(127.0.0.1:39443)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374316206Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"io task","client":"2","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"ds_connection connected from Ok(127.0.0.1:34753)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.374332385Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"io task","client":"0","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375069805Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"client":"2","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375092414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"client":"1","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375103833Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"client":"0","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375377924Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375388004Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"[0]R flush_numbers[0..12]: [132, 139, 117, 129, 135, 138, 139, 137, 116, 134, 134, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375423241Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"downstairs","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"[0]R generation[0..12]: [75, 77, 67, 73, 76, 76, 77, 76, 66, 76, 76, 75]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.37543466Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"downstairs","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.37544635Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"downstairs","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"[1]R flush_numbers[0..12]: [132, 139, 117, 129, 135, 138, 139, 137, 116, 134, 134, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375457399Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"downstairs","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"[1]R generation[0..12]: [75, 77, 67, 73, 76, 76, 77, 76, 66, 76, 76, 75]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375468028Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"downstairs","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375479287Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"downstairs","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"[2]R flush_numbers[0..12]: [132, 139, 117, 129, 135, 138, 139, 137, 116, 134, 134, 132]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375490197Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"downstairs","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"[2]R generation[0..12]: [75, 77, 67, 73, 76, 76, 77, 76, 66, 76, 76, 75]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375502956Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"downstairs","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375514165Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"downstairs","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375523874Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"downstairs","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375533834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"downstairs","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375543543Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"downstairs","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375553232Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"":"downstairs","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375563422Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375572511Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"client":"0","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.37558251Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"client":"1","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.37559241Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"client":"2","":"downstairs","session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375602489Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375611968Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"d4f002cf-e725-4afa-9dd5-b946cfebf9ea is now active with session: 5a0b344e-aeb4-47bd-878f-1ed5a04c1f03","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375622248Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375631297Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"session_id":"5a0b344e-aeb4-47bd-878f-1ed5a04c1f03"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375640586Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445} {"msg":"Activated sub_volume d4f002cf-e725-4afa-9dd5-b946cfebf9ea","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375648936Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.375657485Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.376009941Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"task":"crutest"} 01/30 Write block 544 len 40960 data: 7 7 6 6 8 8 7 6 6 7 02/30 Write block 562 len 12288 data: 6 6 6 03/30 Read block 52 len 28672 04/30 Write block 96 len 12288 data: 7 8 10 05/30 Write block 458 len 12288 data: 10 9 8 06/30 Read block 596 len 16384 07/30 Read block 60 len 40960 08/30 Write block 493 len 12288 data: 8 6 5 09/30 Read block 581 len 16384 10/30 Read block 539 len 20480 11/30 Read block 27 len 4096 12/30 Flush 13/30 Write block 211 len 40960 data: 9 8 7 6 7 7 5 6 6 7 14/30 Write block 355 len 12288 data: 8 9 9 15/30 Write block 340 len 12288 data: 7 7 8 16/30 Write block 558 len 32768 data: 7 8 7 7 7 7 7 7 17/30 Flush 18/30 Flush 19/30 Flush 20/30 Write block 413 len 16384 data: 9 9 11 11 21/30 Flush 22/30 Write block 89 len 24576 data: 4 4 5 7 9 8 23/30 Flush 24/30 Write block 571 len 24576 data: 8 9 10 9 13 12 25/30 Read block 109 len 24576 26/30 Flush 27/30 Write block 449 len 8192 data: 7 7 28/30 Write block 203 len 24576 data: 6 9 10 9 8 11 29/30 Read block 326 len 8192 30/30 Write block 159 len 40960 data: 6 6 5 5 5 6 7 8 11 10 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Write 3 Sent Done Done false 1004 Acked Write 3 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 6 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 1 29 Done 3 29 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:36.3842652Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:36.384543101Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1445,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 132 132 132 F F F 1 020-039 77 77 77 139 139 139 F F F 2 040-059 67 67 67 117 117 117 F F F 3 060-079 73 73 73 129 129 129 F F F 4 080-099 76 78 78 135 145 145 F F F <--- 5 100-119 76 76 76 138 138 138 F F F 6 120-139 77 77 77 139 139 139 F F F 7 140-159 76 76 76 137 137 137 F T T <--- 8 160-179 66 66 66 116 116 116 F T T <--- 9 180-199 76 76 76 134 134 134 F F F 10 200-219 76 78 78 134 141 141 F T T <--- 11 220-239 75 78 78 132 141 141 F F F <--- 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 76 76 76 138 138 138 F F F 16 320-339 70 70 70 123 123 123 F F F 17 340-359 76 78 78 136 141 141 F F F <--- 18 360-379 76 76 76 136 136 136 F F F 19 380-399 73 73 73 129 129 129 F F F 20 400-419 76 78 78 134 144 144 F F F <--- 21 420-439 70 70 70 123 123 123 F F F 22 440-459 77 78 78 139 140 140 F T T <--- 23 460-479 77 78 78 139 140 140 F F F <--- 24 480-499 77 78 78 139 140 140 F F F <--- 25 500-519 77 77 77 139 139 139 F F F 26 520-539 76 76 76 135 135 135 F F F 27 540-559 72 78 78 128 141 141 T F F <--- 28 560-579 72 78 78 128 146 146 T F F <--- 29 580-599 76 76 76 135 135 135 F F F Max gen: 78, Max flush: 146 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 132 132 132 F F F 1 020-039 77 77 77 139 139 139 F F F 2 040-059 67 67 67 117 117 117 F F F 3 060-079 73 73 73 129 129 129 F F F 4 080-099 78 78 78 145 145 145 F F F 5 100-119 76 76 76 138 138 138 F F F 6 120-139 77 77 77 139 139 139 F F F 7 140-159 79 79 79 147 147 147 F F F 8 160-179 79 79 79 147 147 147 F F F 9 180-199 76 76 76 134 134 134 F F F 10 200-219 79 79 79 147 147 147 F F F 11 220-239 78 78 78 141 141 141 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 76 76 76 138 138 138 F F F 16 320-339 70 70 70 123 123 123 F F F 17 340-359 78 78 78 141 141 141 F F F 18 360-379 76 76 76 136 136 136 F F F 19 380-399 73 73 73 129 129 129 F F F 20 400-419 78 78 78 144 144 144 F F F 21 420-439 70 70 70 123 123 123 F F F 22 440-459 79 79 79 147 147 147 F F F 23 460-479 78 78 78 140 140 140 F F F 24 480-499 78 78 78 140 140 140 F F F 25 500-519 77 77 77 139 139 139 F F F 26 520-539 76 76 76 135 135 135 F F F 27 540-559 78 78 78 141 141 141 F F F 28 560-579 78 78 78 146 146 146 F F F 29 580-599 76 76 76 135 135 135 F F F Max gen: 79, Max flush: 147 Begin loop 40 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.633892079Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.642214283Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.642486954Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.642515013Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.642524282Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.642536161Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453} {"msg":"Crucible 25c1d615-5627-4c8a-9a98-aedd225ab6c7 has session id: 4dd6c7b9-0a72-4465-b0d9-f3daea7a610f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.642874728Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"Upstairs opts: Upstairs UUID: 25c1d615-5627-4c8a-9a98-aedd225ab6c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.642898776Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"Crucible stats registered with UUID: 25c1d615-5627-4c8a-9a98-aedd225ab6c7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.642909136Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:40.642922455Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.643102273Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"io task","client":"1","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.64313519Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"io task","client":"0","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.64314695Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"io task","client":"2","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.643252832Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.643277761Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453} {"msg":"25c1d615-5627-4c8a-9a98-aedd225ab6c7 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.643295599Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.643304819Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"client":"0","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.643314758Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"client":"1","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.643324757Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"client":"2","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.643334667Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"io task","client":"2","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.643344936Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"io task","client":"0","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.643355235Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"io task","client":"1","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"ds_connection connected from Ok(127.0.0.1:46149)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.643445249Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"io task","client":"0","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33580)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.643471767Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"io task","client":"2","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61424)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.643483267Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"io task","client":"1","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644037789Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"client":"0","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644124273Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"client":"2","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644137382Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"client":"1","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644351498Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644485868Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"[0]R flush_numbers[0..12]: [132, 139, 117, 129, 145, 138, 139, 147, 147, 134, 147, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644525156Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"downstairs","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"[0]R generation[0..12]: [75, 77, 67, 73, 78, 76, 77, 79, 79, 76, 79, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644536815Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"downstairs","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644548184Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"downstairs","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"[1]R flush_numbers[0..12]: [132, 139, 117, 129, 145, 138, 139, 147, 147, 134, 147, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644559073Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"downstairs","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"[1]R generation[0..12]: [75, 77, 67, 73, 78, 76, 77, 79, 79, 76, 79, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644569563Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"downstairs","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644580752Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"downstairs","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"[2]R flush_numbers[0..12]: [132, 139, 117, 129, 145, 138, 139, 147, 147, 134, 147, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644591761Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"downstairs","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"[2]R generation[0..12]: [75, 77, 67, 73, 78, 76, 77, 79, 79, 76, 79, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644602401Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"downstairs","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.64461349Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"downstairs","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644623419Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"downstairs","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644633398Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"downstairs","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"Next flush: 148","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644643218Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"downstairs","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644652797Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"":"downstairs","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644664966Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644674116Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"client":"0","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644684025Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"client":"1","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644694084Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"client":"2","":"downstairs","session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644704134Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644713573Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"25c1d615-5627-4c8a-9a98-aedd225ab6c7 is now active with session: 4dd6c7b9-0a72-4465-b0d9-f3daea7a610f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644723592Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644732572Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"session_id":"4dd6c7b9-0a72-4465-b0d9-f3daea7a610f"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.644741881Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453} {"msg":"Activated sub_volume 25c1d615-5627-4c8a-9a98-aedd225ab6c7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.64475018Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.64475854Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.645107706Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"task":"crutest"} 01/30 Write block 211 len 28672 data: 10 9 8 7 8 8 6 02/30 Write block 142 len 40960 data: 10 10 10 9 10 10 9 9 10 10 03/30 Write block 63 len 16384 data: 7 8 7 5 04/30 Read block 316 len 8192 05/30 Write block 43 len 40960 data: 7 7 7 7 6 5 2 3 4 3 06/30 Read block 250 len 4096 07/30 Read block 82 len 8192 08/30 Write block 106 len 32768 data: 5 6 8 7 7 7 5 5 09/30 Flush 10/30 Write block 164 len 20480 data: 7 8 9 12 11 11/30 Read block 377 len 16384 12/30 Flush 13/30 Write block 378 len 36864 data: 10 9 8 8 8 8 5 5 4 14/30 Write block 318 len 24576 data: 12 12 11 9 9 9 15/30 Write block 518 len 32768 data: 4 5 6 6 7 6 6 8 16/30 Write block 535 len 8192 data: 6 5 17/30 Read block 507 len 12288 18/30 Read block 127 len 24576 19/30 Write block 172 len 8192 data: 10 11 20/30 Write block 143 len 36864 data: 11 11 10 11 11 10 10 11 11 21/30 Flush 22/30 Write block 74 len 40960 data: 7 6 7 8 9 7 8 7 5 4 23/30 Write block 317 len 4096 data: 13 24/30 Read block 403 len 40960 25/30 Write block 482 len 12288 data: 5 6 6 26/30 Flush 27/30 Write block 428 len 4096 data: 6 28/30 Write block 144 len 40960 data: 12 11 12 12 11 11 12 12 9 6 29/30 Read block 23 len 24576 30/30 Write block 396 len 32768 data: 9 10 10 11 9 8 8 7 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Write 10 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Read 2 Done Done Done false 1007 Acked Write 8 Done Done Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Write 2 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Write 9 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Write 1 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 22 1 24 Done 29 8 29 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:40.653943585Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:40.654242145Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1453,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 132 132 132 F F F 1 020-039 77 77 77 139 139 139 F F F 2 040-059 80 67 80 148 117 148 F T F <--- 3 060-079 80 73 80 151 129 151 F T F <--- 4 080-099 80 78 80 151 145 151 F F F <--- 5 100-119 80 76 80 148 138 148 F T F <--- 6 120-139 77 77 77 139 139 139 F F F 7 140-159 80 79 80 150 147 150 T T T <--- 8 160-179 80 79 80 150 147 150 F F F <--- 9 180-199 76 76 76 134 134 134 F F F 10 200-219 80 79 80 148 147 148 F T F <--- 11 220-239 78 78 78 141 141 141 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 80 76 80 151 138 151 F F F <--- 16 320-339 80 70 80 150 123 150 F F F <--- 17 340-359 78 78 78 141 141 141 F F F 18 360-379 80 76 80 150 136 150 F F F <--- 19 380-399 80 73 80 150 129 150 T F T <--- 20 400-419 78 78 78 144 144 144 T F T <--- 21 420-439 70 70 70 123 123 123 T F T <--- 22 440-459 79 79 79 147 147 147 F F F 23 460-479 78 78 78 140 140 140 F F F 24 480-499 80 78 80 151 140 151 F F F <--- 25 500-519 80 77 80 150 139 150 F F F <--- 26 520-539 80 76 80 150 135 150 F F F <--- 27 540-559 78 78 78 141 141 141 F F F 28 560-579 78 78 78 146 146 146 F F F 29 580-599 76 76 76 135 135 135 F F F Max gen: 80, Max flush: 151 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 132 132 132 F F F 1 020-039 77 77 77 139 139 139 F F F 2 040-059 80 80 80 148 148 148 F F F 3 060-079 80 80 80 151 151 151 F F F 4 080-099 80 80 80 151 151 151 F F F 5 100-119 80 80 80 148 148 148 F F F 6 120-139 77 77 77 139 139 139 F F F 7 140-159 81 81 81 152 152 152 F F F 8 160-179 80 80 80 150 150 150 F F F 9 180-199 76 76 76 134 134 134 F F F 10 200-219 80 80 80 148 148 148 F F F 11 220-239 78 78 78 141 141 141 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 F F F 14 280-299 72 72 72 128 128 128 F F F 15 300-319 80 80 80 151 151 151 F F F 16 320-339 80 80 80 150 150 150 F F F 17 340-359 78 78 78 141 141 141 F F F 18 360-379 80 80 80 150 150 150 F F F 19 380-399 81 81 81 152 152 152 F F F 20 400-419 81 81 81 152 152 152 F F F 21 420-439 81 81 81 152 152 152 F F F 22 440-459 79 79 79 147 147 147 F F F 23 460-479 78 78 78 140 140 140 F F F 24 480-499 80 80 80 151 151 151 F F F 25 500-519 80 80 80 150 150 150 F F F 26 520-539 80 80 80 150 150 150 F F F 27 540-559 78 78 78 141 141 141 F F F 28 560-579 78 78 78 146 146 146 F F F 29 580-599 76 76 76 135 135 135 F F F Max gen: 81, Max flush: 152 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.900994529Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.909142304Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.909417746Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.909446234Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.909458003Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.909469832Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461} {"msg":"Crucible 54f6d36c-afd7-467b-bd4b-b1a653e72afb has session id: 49964bf8-6201-4145-9415-0871443bb6df","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.90979858Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"Upstairs opts: Upstairs UUID: 54f6d36c-afd7-467b-bd4b-b1a653e72afb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.909833517Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"Crucible stats registered with UUID: 54f6d36c-afd7-467b-bd4b-b1a653e72afb","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.909845067Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:44.909854846Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910030644Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"io task","client":"2","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910060412Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"io task","client":"0","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910071591Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"io task","client":"1","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910161455Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910179684Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461} {"msg":"54f6d36c-afd7-467b-bd4b-b1a653e72afb active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910205592Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910214891Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"client":"0","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910225061Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"client":"1","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.91023512Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"client":"2","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910247699Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"io task","client":"2","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910303545Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"io task","client":"1","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"ds_connection connected from Ok(127.0.0.1:47960)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910315015Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"io task","client":"2","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910325254Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"io task","client":"0","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"ds_connection connected from Ok(127.0.0.1:47432)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910368391Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"io task","client":"0","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"ds_connection connected from Ok(127.0.0.1:44721)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.910394099Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"io task","client":"1","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911064724Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"client":"0","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911081442Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"client":"1","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911097231Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"client":"2","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911373863Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.91141055Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"[0]R flush_numbers[0..12]: [132, 139, 148, 151, 151, 148, 139, 152, 150, 134, 148, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911448827Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"downstairs","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"[0]R generation[0..12]: [75, 77, 80, 80, 80, 80, 77, 81, 80, 76, 80, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911460237Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"downstairs","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911471736Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"downstairs","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"[1]R flush_numbers[0..12]: [132, 139, 148, 151, 151, 148, 139, 152, 150, 134, 148, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911484505Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"downstairs","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"[1]R generation[0..12]: [75, 77, 80, 80, 80, 80, 77, 81, 80, 76, 80, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911495324Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"downstairs","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911506644Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"downstairs","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"[2]R flush_numbers[0..12]: [132, 139, 148, 151, 151, 148, 139, 152, 150, 134, 148, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911517463Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"downstairs","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"[2]R generation[0..12]: [75, 77, 80, 80, 80, 80, 77, 81, 80, 76, 80, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911527992Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"downstairs","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911539191Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"downstairs","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911548931Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"downstairs","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.91155892Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"downstairs","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911568829Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"downstairs","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911578649Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"":"downstairs","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911588788Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911597897Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"client":"0","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911607827Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"client":"1","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911617856Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"client":"2","":"downstairs","session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911628045Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911638565Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"54f6d36c-afd7-467b-bd4b-b1a653e72afb is now active with session: 49964bf8-6201-4145-9415-0871443bb6df","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911648794Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911657813Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"session_id":"49964bf8-6201-4145-9415-0871443bb6df"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911667253Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461} {"msg":"Activated sub_volume 54f6d36c-afd7-467b-bd4b-b1a653e72afb","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911675472Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.911683821Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.912031718Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"task":"crutest"} 01/30 Write block 546 len 28672 data: 7 7 9 9 8 7 7 02/30 Write block 299 len 36864 data: 7 6 6 6 6 8 8 10 11 03/30 Write block 272 len 36864 data: 3 3 2 3 4 3 3 3 3 04/30 Read block 140 len 4096 05/30 Write block 89 len 28672 data: 5 5 6 8 10 9 8 06/30 Write block 0 len 4096 data: 3 07/30 Write block 154 len 12288 data: 6 5 5 08/30 Read block 407 len 36864 09/30 Read block 219 len 8192 10/30 Read block 202 len 24576 11/30 Read block 212 len 32768 12/30 Read block 259 len 36864 13/30 Read block 461 len 36864 14/30 Write block 437 len 16384 data: 7 7 6 7 15/30 Write block 88 len 8192 data: 5 6 16/30 Write block 439 len 24576 data: 7 8 8 7 6 4 17/30 Write block 484 len 20480 data: 7 8 8 10 8 18/30 Read block 563 len 8192 19/30 Read block 430 len 12288 20/30 Read block 9 len 24576 21/30 Write block 206 len 32768 data: 10 9 12 9 10 11 10 9 22/30 Write block 316 len 12288 data: 13 14 13 23/30 Read block 529 len 28672 24/30 Read block 486 len 16384 25/30 Read block 219 len 28672 26/30 Read block 380 len 28672 27/30 Read block 267 len 24576 28/30 Write block 372 len 4096 data: 8 29/30 Write block 374 len 28672 data: 8 10 9 9 11 10 9 30/30 Write block 470 len 36864 data: 9 8 6 6 4 4 4 4 6 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Write 3 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 2 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 8 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Write 2 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 2 Done Sent Done false 1018 Acked Read 3 Done Sent Done false 1019 Acked Read 6 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 7 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 27 2 31 Done 28 3 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:44.919759212Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:44.920057552Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1461,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 132 132 132 T F T <--- 1 020-039 77 77 77 139 139 139 F F F 2 040-059 80 80 80 148 148 148 F F F 3 060-079 80 80 80 151 151 151 F F F 4 080-099 80 80 80 151 151 151 T F T <--- 5 100-119 80 80 80 148 148 148 F F F 6 120-139 77 77 77 139 139 139 F F F 7 140-159 81 81 81 152 152 152 T F T <--- 8 160-179 80 80 80 150 150 150 F F F 9 180-199 76 76 76 134 134 134 F F F 10 200-219 80 80 80 148 148 148 T F T <--- 11 220-239 78 78 78 141 141 141 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 66 66 66 116 116 116 T T T 14 280-299 72 72 72 128 128 128 T T T 15 300-319 80 80 80 151 151 151 T T T 16 320-339 80 80 80 150 150 150 F F F 17 340-359 78 78 78 141 141 141 F F F 18 360-379 80 80 80 150 150 150 T F T <--- 19 380-399 81 81 81 152 152 152 T F T <--- 20 400-419 81 81 81 152 152 152 F F F 21 420-439 81 81 81 152 152 152 T F T <--- 22 440-459 79 79 79 147 147 147 T F T <--- 23 460-479 78 78 78 140 140 140 T F T <--- 24 480-499 80 80 80 151 151 151 T F T <--- 25 500-519 80 80 80 150 150 150 F F F 26 520-539 80 80 80 150 150 150 F F F 27 540-559 78 78 78 141 141 141 T T T 28 560-579 78 78 78 146 146 146 F F F 29 580-599 76 76 76 135 135 135 F F F Max gen: 81, Max flush: 152 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 153 153 153 F F F 1 020-039 77 77 77 139 139 139 F F F 2 040-059 80 80 80 148 148 148 F F F 3 060-079 80 80 80 151 151 151 F F F 4 080-099 82 82 82 153 153 153 F F F 5 100-119 80 80 80 148 148 148 F F F 6 120-139 77 77 77 139 139 139 F F F 7 140-159 82 82 82 153 153 153 F F F 8 160-179 80 80 80 150 150 150 F F F 9 180-199 76 76 76 134 134 134 F F F 10 200-219 82 82 82 153 153 153 F F F 11 220-239 78 78 78 141 141 141 F F F 12 240-259 66 66 66 116 116 116 F F F 13 260-279 82 82 82 153 153 153 F F F 14 280-299 82 82 82 153 153 153 F F F 15 300-319 82 82 82 153 153 153 F F F 16 320-339 80 80 80 150 150 150 F F F 17 340-359 78 78 78 141 141 141 F F F 18 360-379 82 82 82 153 153 153 F F F 19 380-399 82 82 82 153 153 153 F F F 20 400-419 81 81 81 152 152 152 F F F 21 420-439 82 82 82 153 153 153 F F F 22 440-459 82 82 82 153 153 153 F F F 23 460-479 82 82 82 153 153 153 F F F 24 480-499 82 82 82 153 153 153 F F F 25 500-519 80 80 80 150 150 150 F F F 26 520-539 80 80 80 150 150 150 F F F 27 540-559 82 82 82 153 153 153 F F F 28 560-579 78 78 78 146 146 146 F F F 29 580-599 76 76 76 135 135 135 F F F Max gen: 82, Max flush: 153 Begin loop 42 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.173370071Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.181502597Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.181803807Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.181830695Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.181839704Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.181851444Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469} {"msg":"Crucible 77ea7624-5aaa-4cb0-b901-a99da03cbd75 has session id: b1990209-554e-4265-957c-879b8a3cd527","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182217249Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"Upstairs opts: Upstairs UUID: 77ea7624-5aaa-4cb0-b901-a99da03cbd75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182240327Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"Crucible stats registered with UUID: 77ea7624-5aaa-4cb0-b901-a99da03cbd75","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182253126Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:49.182265745Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182441073Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182458122Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182468602Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182596933Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182615292Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469} {"msg":"77ea7624-5aaa-4cb0-b901-a99da03cbd75 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182655279Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182665028Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"client":"0","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182675168Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"client":"1","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182685177Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"client":"2","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182695266Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182705785Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182715995Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"ds_connection connected from Ok(127.0.0.1:54568)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182757762Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"io task","client":"2","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"ds_connection connected from Ok(127.0.0.1:54756)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182795229Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"io task","client":"0","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"ds_connection connected from Ok(127.0.0.1:36090)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.182827957Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"io task","client":"1","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.183480383Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"client":"0","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.183502271Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"client":"1","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.183539659Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"client":"2","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.183836199Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.183885265Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"[0]R flush_numbers[0..12]: [153, 139, 148, 151, 153, 148, 139, 153, 150, 134, 153, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.183941881Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"downstairs","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"[0]R generation[0..12]: [82, 77, 80, 80, 82, 80, 77, 82, 80, 76, 82, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.183953201Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"downstairs","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.18396453Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"downstairs","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"[1]R flush_numbers[0..12]: [153, 139, 148, 151, 153, 148, 139, 153, 150, 134, 153, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.183975759Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"downstairs","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"[1]R generation[0..12]: [82, 77, 80, 80, 82, 80, 77, 82, 80, 76, 82, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.183986468Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"downstairs","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.183997618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"downstairs","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"[2]R flush_numbers[0..12]: [153, 139, 148, 151, 153, 148, 139, 153, 150, 134, 153, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184008557Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"downstairs","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"[2]R generation[0..12]: [82, 77, 80, 80, 82, 80, 77, 82, 80, 76, 82, 78]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184019116Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"downstairs","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184032085Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"downstairs","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"Max found gen is 83","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184041755Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"downstairs","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"Generation requested: 84 >= found:83","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184051774Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"downstairs","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"Next flush: 154","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184061423Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"downstairs","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184071033Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"":"downstairs","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184081002Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184090021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"client":"0","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184100021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"client":"1","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.18410993Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"client":"2","":"downstairs","session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184120109Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184129459Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"77ea7624-5aaa-4cb0-b901-a99da03cbd75 is now active with session: b1990209-554e-4265-957c-879b8a3cd527","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184139658Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184149037Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184156997Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"session_id":"b1990209-554e-4265-957c-879b8a3cd527"} {"msg":"Activated sub_volume 77ea7624-5aaa-4cb0-b901-a99da03cbd75","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184166536Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184174975Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.184461126Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"task":"crutest"} 01/30 Write block 349 len 32768 data: 7 6 5 6 7 7 9 10 02/30 Write block 311 len 24576 data: 11 13 12 13 11 14 03/30 Read block 236 len 12288 04/30 Read block 385 len 36864 05/30 Write block 528 len 28672 data: 6 7 6 7 8 7 5 06/30 Write block 111 len 12288 data: 8 6 6 07/30 Read block 222 len 4096 08/30 Write block 542 len 24576 data: 6 6 8 8 8 8 09/30 Read block 70 len 24576 10/30 Write block 309 len 16384 data: 9 9 12 14 11/30 Write block 254 len 32768 data: 3 4 4 6 6 5 5 5 12/30 Write block 173 len 20480 data: 12 11 8 6 3 13/30 Write block 118 len 12288 data: 5 5 4 14/30 Read block 127 len 16384 15/30 Write block 442 len 32768 data: 8 7 5 4 3 5 7 8 16/30 Flush 17/30 Write block 341 len 20480 data: 8 9 8 9 9 18/30 Write block 465 len 32768 data: 12 12 8 8 9 10 9 7 19/30 Write block 225 len 28672 data: 6 8 9 11 7 7 9 20/30 Read block 107 len 8192 21/30 Read block 63 len 16384 22/30 Flush 23/30 Write block 492 len 40960 data: 7 9 7 6 6 6 4 5 5 5 24/30 Read block 400 len 8192 25/30 Write block 498 len 24576 data: 5 6 6 6 6 6 26/30 Read block 292 len 24576 27/30 Read block 214 len 12288 28/30 Write block 331 len 24576 data: 11 12 11 9 7 7 29/30 Write block 359 len 12288 data: 11 10 13 30/30 Write block 389 len 32768 data: 2 5 6 5 6 9 10 10 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Sent false 1001 Acked Write 6 Done Done Sent false 1002 Acked Read 3 Done Done Sent false 1003 Acked Read 9 Done Done Sent false 1004 Acked Write 7 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Read 4 Done Done Sent false 1014 Acked Write 8 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 5 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Write 7 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Write 6 Done Done Sent false 1025 Acked Read 6 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 30 36 Done 27 27 0 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:49.192702865Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:49.192965307Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1469,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 153 153 153 F F F 1 020-039 77 77 77 139 139 139 F F F 2 040-059 80 80 80 148 148 148 F F F 3 060-079 80 80 80 151 151 151 F F F 4 080-099 82 82 82 153 153 153 F F F 5 100-119 84 84 80 154 154 148 F F F <--- 6 120-139 84 84 77 154 154 139 F F F <--- 7 140-159 82 82 82 153 153 153 F F F 8 160-179 84 84 80 154 154 150 F F F <--- 9 180-199 76 76 76 134 134 134 F F F 10 200-219 82 82 82 153 153 153 F F F 11 220-239 84 84 78 155 155 141 F F F <--- 12 240-259 84 84 66 154 154 116 F F F <--- 13 260-279 84 84 82 154 154 153 F F F <--- 14 280-299 82 82 82 153 153 153 F F F 15 300-319 84 84 82 154 154 153 F F F <--- 16 320-339 80 80 80 150 150 150 T T F <--- 17 340-359 84 84 78 155 155 141 T T F <--- 18 360-379 82 82 82 153 153 153 T T F <--- 19 380-399 82 82 82 153 153 153 T T F <--- 20 400-419 81 81 81 152 152 152 F F F 21 420-439 82 82 82 153 153 153 F F F 22 440-459 84 84 82 154 154 153 F F F <--- 23 460-479 84 84 82 155 155 153 F F F <--- 24 480-499 82 82 82 153 153 153 T T F <--- 25 500-519 80 80 80 150 150 150 T T F <--- 26 520-539 84 84 80 154 154 150 F F F <--- 27 540-559 84 84 82 154 154 153 F F F <--- 28 560-579 78 78 78 146 146 146 F F F 29 580-599 76 76 76 135 135 135 F F F Max gen: 84, Max flush: 155 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 153 153 153 F F F 1 020-039 77 77 77 139 139 139 F F F 2 040-059 80 80 80 148 148 148 F F F 3 060-079 80 80 80 151 151 151 F F F 4 080-099 82 82 82 153 153 153 F F F 5 100-119 84 84 84 154 154 154 F F F 6 120-139 84 84 84 154 154 154 F F F 7 140-159 82 82 82 153 153 153 F F F 8 160-179 84 84 84 154 154 154 F F F 9 180-199 76 76 76 134 134 134 F F F 10 200-219 82 82 82 153 153 153 F F F 11 220-239 84 84 84 155 155 155 F F F 12 240-259 84 84 84 154 154 154 F F F 13 260-279 84 84 84 154 154 154 F F F 14 280-299 82 82 82 153 153 153 F F F 15 300-319 84 84 84 154 154 154 F F F 16 320-339 85 85 85 156 156 156 F F F 17 340-359 85 85 85 156 156 156 F F F 18 360-379 85 85 85 156 156 156 F F F 19 380-399 85 85 85 156 156 156 F F F 20 400-419 81 81 81 152 152 152 F F F 21 420-439 82 82 82 153 153 153 F F F 22 440-459 84 84 84 154 154 154 F F F 23 460-479 84 84 84 155 155 155 F F F 24 480-499 85 85 85 156 156 156 F F F 25 500-519 85 85 85 156 156 156 F F F 26 520-539 84 84 84 154 154 154 F F F 27 540-559 84 84 84 154 154 154 F F F 28 560-579 78 78 78 146 146 146 F F F 29 580-599 76 76 76 135 135 135 F F F Max gen: 85, Max flush: 156 Begin loop 43 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.443988372Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.452166406Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.452437587Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.452468155Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.452477525Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.452489524Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477} {"msg":"Crucible 2073cfbe-ddc3-4a30-88b5-8ef6968291c9 has session id: 546b874b-897e-418e-91dc-7e6794027b45","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.452816651Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"Upstairs opts: Upstairs UUID: 2073cfbe-ddc3-4a30-88b5-8ef6968291c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.452846939Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"Crucible stats registered with UUID: 2073cfbe-ddc3-4a30-88b5-8ef6968291c9","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.452858539Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:53.452868508Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453031767Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"io task","client":"0","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453062845Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"io task","client":"2","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453076754Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"io task","client":"1","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453192496Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453209365Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477} {"msg":"2073cfbe-ddc3-4a30-88b5-8ef6968291c9 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453222314Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453231523Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"client":"0","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453241563Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"client":"1","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453251442Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"client":"2","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453261351Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"io task","client":"2","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453271711Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"io task","client":"0","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.45328188Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"io task","client":"1","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"ds_connection connected from Ok(127.0.0.1:35373)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453375933Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"io task","client":"2","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"ds_connection connected from Ok(127.0.0.1:40433)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453402142Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"io task","client":"1","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"ds_connection connected from Ok(127.0.0.1:61974)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.453413581Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"io task","client":"0","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454118123Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"client":"0","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454132352Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"client":"1","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454143541Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"client":"2","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454435061Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454476269Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"[0]R flush_numbers[0..12]: [153, 139, 148, 151, 153, 154, 154, 153, 154, 134, 153, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454523155Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"downstairs","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"[0]R generation[0..12]: [82, 77, 80, 80, 82, 84, 84, 82, 84, 76, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454534495Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"downstairs","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454545894Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"downstairs","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"[1]R flush_numbers[0..12]: [153, 139, 148, 151, 153, 154, 154, 153, 154, 134, 153, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454556853Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"downstairs","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"[1]R generation[0..12]: [82, 77, 80, 80, 82, 84, 84, 82, 84, 76, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454567422Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"downstairs","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454578602Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"downstairs","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"[2]R flush_numbers[0..12]: [153, 139, 148, 151, 153, 154, 154, 153, 154, 134, 153, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454589351Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"downstairs","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"[2]R generation[0..12]: [82, 77, 80, 80, 82, 84, 84, 82, 84, 76, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.45459988Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"downstairs","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454611359Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"downstairs","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454621149Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"downstairs","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454631138Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"downstairs","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454640847Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"downstairs","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454652277Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"":"downstairs","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454662376Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454671395Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"client":"0","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454681325Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"client":"1","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454691214Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"client":"2","":"downstairs","session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454701313Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454710903Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"2073cfbe-ddc3-4a30-88b5-8ef6968291c9 is now active with session: 546b874b-897e-418e-91dc-7e6794027b45","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454721112Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454730531Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454738711Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"session_id":"546b874b-897e-418e-91dc-7e6794027b45"} {"msg":"Activated sub_volume 2073cfbe-ddc3-4a30-88b5-8ef6968291c9","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.45474829Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.454756779Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.455050389Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"task":"crutest"} 01/30 Write block 335 len 8192 data: 8 8 02/30 Write block 461 len 40960 data: 7 6 9 10 13 13 9 9 10 11 03/30 Flush 04/30 Write block 102 len 36864 data: 5 6 7 5 6 7 9 8 8 05/30 Flush 06/30 Write block 378 len 8192 data: 12 11 07/30 Write block 570 len 20480 data: 10 9 10 11 10 08/30 Write block 11 len 20480 data: 7 7 9 6 5 09/30 Write block 263 len 28672 data: 6 6 4 4 3 5 5 10/30 Write block 503 len 32768 data: 7 8 10 8 11 10 10 9 11/30 Read block 160 len 8192 12/30 Read block 561 len 36864 13/30 Write block 191 len 36864 data: 8 8 7 7 8 5 4 6 8 14/30 Write block 559 len 28672 data: 9 8 8 8 8 8 8 15/30 Read block 90 len 4096 16/30 Write block 401 len 40960 data: 9 9 8 7 7 7 7 7 5 5 17/30 Write block 90 len 40960 data: 6 7 9 11 10 9 8 9 11 10 18/30 Read block 577 len 20480 19/30 Write block 436 len 40960 data: 8 8 8 8 9 9 9 8 6 5 20/30 Read block 519 len 28672 21/30 Write block 333 len 16384 data: 12 10 9 9 22/30 Write block 440 len 12288 data: 10 10 10 23/30 Read block 583 len 36864 24/30 Write block 534 len 36864 data: 6 7 6 6 6 6 6 4 7 25/30 Write block 35 len 16384 data: 8 8 6 7 26/30 Write block 140 len 24576 data: 8 9 11 12 13 12 27/30 Write block 74 len 16384 data: 8 7 8 9 28/30 Write block 450 len 12288 data: 8 4 4 29/30 Write block 46 len 32768 data: 8 7 6 3 4 5 4 2 30/30 Read block 301 len 24576 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 10 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 9 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 2 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Read 9 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Write 10 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 1 30 Done 30 1 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) None Some(JobId(1004)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:53.462419328Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:53.462702049Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1477,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 153 153 153 T F T <--- 1 020-039 77 77 77 139 139 139 T F T <--- 2 040-059 80 80 80 148 148 148 T F T <--- 3 060-079 80 80 80 151 151 151 T F T <--- 4 080-099 82 82 82 153 153 153 T F T <--- 5 100-119 86 84 86 158 154 158 F F F <--- 6 120-139 84 84 84 154 154 154 F F F 7 140-159 82 82 82 153 153 153 T F T <--- 8 160-179 84 84 84 154 154 154 F F F 9 180-199 76 76 76 134 134 134 T F T <--- 10 200-219 82 82 82 153 153 153 F F F 11 220-239 84 84 84 155 155 155 F F F 12 240-259 84 84 84 154 154 154 F F F 13 260-279 84 84 84 154 154 154 T F T <--- 14 280-299 82 82 82 153 153 153 F F F 15 300-319 84 84 84 154 154 154 F F F 16 320-339 86 85 86 157 156 157 T T T <--- 17 340-359 85 85 85 156 156 156 F F F 18 360-379 85 85 85 156 156 156 T F T <--- 19 380-399 85 85 85 156 156 156 F F F 20 400-419 81 81 81 152 152 152 T F T <--- 21 420-439 82 82 82 153 153 153 T F T <--- 22 440-459 84 84 84 154 154 154 T F T <--- 23 460-479 86 84 86 157 155 157 F F F <--- 24 480-499 85 85 85 156 156 156 F F F 25 500-519 85 85 85 156 156 156 T F T <--- 26 520-539 84 84 84 154 154 154 T F T <--- 27 540-559 84 84 84 154 154 154 T F T <--- 28 560-579 78 78 78 146 146 146 T F T <--- 29 580-599 76 76 76 135 135 135 F F F Max gen: 86, Max flush: 158 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 159 159 159 F F F 1 020-039 87 87 87 159 159 159 F F F 2 040-059 87 87 87 159 159 159 F F F 3 060-079 87 87 87 159 159 159 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 86 86 86 158 158 158 F F F 6 120-139 84 84 84 154 154 154 F F F 7 140-159 87 87 87 159 159 159 F F F 8 160-179 84 84 84 154 154 154 F F F 9 180-199 87 87 87 159 159 159 F F F 10 200-219 82 82 82 153 153 153 F F F 11 220-239 84 84 84 155 155 155 F F F 12 240-259 84 84 84 154 154 154 F F F 13 260-279 87 87 87 159 159 159 F F F 14 280-299 82 82 82 153 153 153 F F F 15 300-319 84 84 84 154 154 154 F F F 16 320-339 87 87 87 159 159 159 F F F 17 340-359 85 85 85 156 156 156 F F F 18 360-379 87 87 87 159 159 159 F F F 19 380-399 85 85 85 156 156 156 F F F 20 400-419 87 87 87 159 159 159 F F F 21 420-439 87 87 87 159 159 159 F F F 22 440-459 87 87 87 159 159 159 F F F 23 460-479 86 86 86 157 157 157 F F F 24 480-499 85 85 85 156 156 156 F F F 25 500-519 87 87 87 159 159 159 F F F 26 520-539 87 87 87 159 159 159 F F F 27 540-559 87 87 87 159 159 159 F F F 28 560-579 87 87 87 159 159 159 F F F 29 580-599 76 76 76 135 135 135 F F F Max gen: 87, Max flush: 159 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.728641529Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:57.735748125Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.735814221Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.742766937Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.74301697Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743041109Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743049968Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743061687Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485} {"msg":"Crucible 255481e6-8dae-46c6-8c6a-f31c694e8d74 has session id: 6ea3e604-6444-4c97-81fd-65ebb17c4be7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743444821Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"Upstairs opts: Upstairs UUID: 255481e6-8dae-46c6-8c6a-f31c694e8d74, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743475109Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"Crucible stats registered with UUID: 255481e6-8dae-46c6-8c6a-f31c694e8d74","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743486779Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:57.743496488Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743663926Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"io task","client":"0","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743680855Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"io task","client":"1","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743691365Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"io task","client":"2","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743780709Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743806357Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485} {"msg":"255481e6-8dae-46c6-8c6a-f31c694e8d74 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743825975Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743835315Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"client":"0","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743845364Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"client":"1","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743855333Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"client":"2","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743865173Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"io task","client":"2","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743875422Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"io task","client":"0","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743885691Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"io task","client":"1","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62889)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.743979015Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"io task","client":"2","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"ds_connection connected from Ok(127.0.0.1:44212)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.744005083Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"io task","client":"1","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46131)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.744062909Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"io task","client":"0","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.744736084Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"client":"1","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.744748893Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"client":"2","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.744760012Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"client":"0","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745037003Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745073071Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"[0]R flush_numbers[0..12]: [159, 159, 159, 159, 159, 158, 154, 159, 154, 159, 153, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745111528Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"downstairs","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"[0]R generation[0..12]: [87, 87, 87, 87, 87, 86, 84, 87, 84, 87, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745122887Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"downstairs","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745134216Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"downstairs","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"[1]R flush_numbers[0..12]: [159, 159, 159, 159, 159, 158, 154, 159, 154, 159, 153, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745145066Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"downstairs","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"[1]R generation[0..12]: [87, 87, 87, 87, 87, 86, 84, 87, 84, 87, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745155755Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"downstairs","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745166824Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"downstairs","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"[2]R flush_numbers[0..12]: [159, 159, 159, 159, 159, 158, 154, 159, 154, 159, 153, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745177603Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"downstairs","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"[2]R generation[0..12]: [87, 87, 87, 87, 87, 86, 84, 87, 84, 87, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745188263Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"downstairs","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745199342Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"downstairs","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745209101Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"downstairs","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745219061Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"downstairs","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.74522873Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"downstairs","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745238559Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"":"downstairs","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745248569Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745257598Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"client":"0","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745269367Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"client":"1","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745279407Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"client":"2","":"downstairs","session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745289606Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745298955Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"255481e6-8dae-46c6-8c6a-f31c694e8d74 is now active with session: 6ea3e604-6444-4c97-81fd-65ebb17c4be7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745309085Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745318104Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"session_id":"6ea3e604-6444-4c97-81fd-65ebb17c4be7"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745327533Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485} {"msg":"Activated sub_volume 255481e6-8dae-46c6-8c6a-f31c694e8d74","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745335823Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745344212Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.745682729Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"task":"crutest"} 01/30 Write block 370 len 36864 data: 7 6 9 8 9 11 10 10 13 02/30 Write block 426 len 36864 data: 6 7 7 5 6 8 8 8 7 03/30 Write block 55 len 16384 data: 6 7 8 7 04/30 Write block 434 len 24576 data: 8 9 9 9 9 9 05/30 Write block 417 len 12288 data: 9 8 8 06/30 Read block 179 len 8192 07/30 Read block 31 len 24576 08/30 Read block 100 len 28672 09/30 Write block 246 len 20480 data: 9 7 4 4 3 10/30 Write block 442 len 4096 data: 11 11/30 Read block 561 len 8192 12/30 Read block 350 len 24576 13/30 Read block 236 len 8192 14/30 Read block 549 len 16384 15/30 Read block 520 len 24576 16/30 Write block 258 len 32768 data: 7 6 6 6 6 7 7 5 17/30 Read block 65 len 36864 18/30 Write block 298 len 28672 data: 9 8 7 7 7 7 9 19/30 Write block 382 len 24576 data: 9 9 6 6 5 2 20/30 Write block 42 len 4096 data: 8 21/30 Write block 385 len 20480 data: 7 6 3 2 3 22/30 Write block 101 len 4096 data: 6 23/30 Write block 240 len 28672 data: 5 6 8 8 8 7 10 24/30 Read block 258 len 40960 25/30 Flush 26/30 Write block 435 len 4096 data: 10 27/30 Read block 433 len 20480 28/30 Read block 199 len 28672 29/30 Write block 183 len 12288 data: 4 5 7 30/30 Read block 374 len 24576 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Write 4 Sent Done Done false 1003 Acked Write 6 Sent Done Done false 1004 Acked Write 3 Sent Done Done false 1005 Acked Read 2 Sent Done Done false 1006 Acked Read 6 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Write 1 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Read 6 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Write 6 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Write 1 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Read 10 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 7 Sent Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 1 29 Done 2 30 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:41:57.753262453Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:41:57.753553114Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1485,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 159 159 159 F F F 1 020-039 87 87 87 159 159 159 F F F 2 040-059 87 88 88 159 160 160 F F F <--- 3 060-079 87 87 87 159 159 159 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 86 88 88 158 160 160 F F F <--- 6 120-139 84 84 84 154 154 154 F F F 7 140-159 87 87 87 159 159 159 F F F 8 160-179 84 84 84 154 154 154 F F F 9 180-199 87 87 87 159 159 159 F T T <--- 10 200-219 82 82 82 153 153 153 F F F 11 220-239 84 84 84 155 155 155 F F F 12 240-259 84 88 88 154 160 160 F F F <--- 13 260-279 87 88 88 159 160 160 F F F <--- 14 280-299 82 88 88 153 160 160 F F F <--- 15 300-319 84 88 88 154 160 160 F F F <--- 16 320-339 87 87 87 159 159 159 F F F 17 340-359 85 85 85 156 156 156 F F F 18 360-379 87 88 88 159 160 160 T F F <--- 19 380-399 85 88 88 156 160 160 F F F <--- 20 400-419 87 88 88 159 160 160 F F F <--- 21 420-439 87 88 88 159 160 160 T T T <--- 22 440-459 87 88 88 159 160 160 F F F <--- 23 460-479 86 86 86 157 157 157 F F F 24 480-499 85 85 85 156 156 156 F F F 25 500-519 87 87 87 159 159 159 F F F 26 520-539 87 87 87 159 159 159 F F F 27 540-559 87 87 87 159 159 159 F F F 28 560-579 87 87 87 159 159 159 F F F 29 580-599 76 76 76 135 135 135 F F F Max gen: 88, Max flush: 160 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 159 159 159 F F F 1 020-039 87 87 87 159 159 159 F F F 2 040-059 88 88 88 160 160 160 F F F 3 060-079 87 87 87 159 159 159 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 88 88 88 160 160 160 F F F 6 120-139 84 84 84 154 154 154 F F F 7 140-159 87 87 87 159 159 159 F F F 8 160-179 84 84 84 154 154 154 F F F 9 180-199 89 89 89 161 161 161 F F F 10 200-219 82 82 82 153 153 153 F F F 11 220-239 84 84 84 155 155 155 F F F 12 240-259 88 88 88 160 160 160 F F F 13 260-279 88 88 88 160 160 160 F F F 14 280-299 88 88 88 160 160 160 F F F 15 300-319 88 88 88 160 160 160 F F F 16 320-339 87 87 87 159 159 159 F F F 17 340-359 85 85 85 156 156 156 F F F 18 360-379 88 88 88 160 160 160 F F F 19 380-399 88 88 88 160 160 160 F F F 20 400-419 88 88 88 160 160 160 F F F 21 420-439 89 89 89 161 161 161 F F F 22 440-459 88 88 88 160 160 160 F F F 23 460-479 86 86 86 157 157 157 F F F 24 480-499 85 85 85 156 156 156 F F F 25 500-519 87 87 87 159 159 159 F F F 26 520-539 87 87 87 159 159 159 F F F 27 540-559 87 87 87 159 159 159 F F F 28 560-579 87 87 87 159 159 159 F F F 29 580-599 76 76 76 135 135 135 F F F Max gen: 89, Max flush: 161 Begin loop 45 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.002463782Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:02.009729318Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.009798223Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.0167407Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.016996483Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.01704215Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017055289Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017067678Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493} {"msg":"Crucible a6076425-d60d-40c7-b719-0f0dcba812fd has session id: 21d44a2e-975f-45cb-a0bc-67276baeba06","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017358538Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"Upstairs opts: Upstairs UUID: a6076425-d60d-40c7-b719-0f0dcba812fd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017412595Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"Crucible stats registered with UUID: a6076425-d60d-40c7-b719-0f0dcba812fd","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017423764Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:02.017435673Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017610921Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"io task","client":"0","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.01762775Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"io task","client":"1","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017638259Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"io task","client":"2","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017763271Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017788979Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493} {"msg":"a6076425-d60d-40c7-b719-0f0dcba812fd active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017807408Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017816587Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"client":"0","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017826707Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"client":"1","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017836596Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"client":"2","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017848195Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"io task","client":"2","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017866354Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"io task","client":"1","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017876833Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"io task","client":"0","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"ds_connection connected from Ok(127.0.0.1:60109)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.01793004Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"io task","client":"2","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"ds_connection connected from Ok(127.0.0.1:49346)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017941369Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"io task","client":"1","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"ds_connection connected from Ok(127.0.0.1:33171)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.017951968Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"io task","client":"0","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.018670959Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"client":"2","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.018697777Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"client":"0","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.018734695Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"client":"1","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019035174Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019066992Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"[0]R flush_numbers[0..12]: [159, 159, 160, 159, 159, 160, 154, 159, 154, 161, 153, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019163516Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"downstairs","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"[0]R generation[0..12]: [87, 87, 88, 87, 87, 88, 84, 87, 84, 89, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019175845Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"downstairs","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019187404Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"downstairs","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"[1]R flush_numbers[0..12]: [159, 159, 160, 159, 159, 160, 154, 159, 154, 161, 153, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019201213Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"downstairs","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"[1]R generation[0..12]: [87, 87, 88, 87, 87, 88, 84, 87, 84, 89, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019212022Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"downstairs","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019223202Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"downstairs","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"[2]R flush_numbers[0..12]: [159, 159, 160, 159, 159, 160, 154, 159, 154, 161, 153, 155]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019234001Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"downstairs","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"[2]R generation[0..12]: [87, 87, 88, 87, 87, 88, 84, 87, 84, 89, 82, 84]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.01924472Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"downstairs","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019256229Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"downstairs","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019265989Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"downstairs","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019275968Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"downstairs","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019285547Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"downstairs","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019295257Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"":"downstairs","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019305446Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019314635Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"client":"0","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019325065Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"client":"1","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019335254Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"client":"2","":"downstairs","session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019345553Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019357572Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"a6076425-d60d-40c7-b719-0f0dcba812fd is now active with session: 21d44a2e-975f-45cb-a0bc-67276baeba06","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019367862Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019377231Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019385241Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"session_id":"21d44a2e-975f-45cb-a0bc-67276baeba06"} {"msg":"Activated sub_volume a6076425-d60d-40c7-b719-0f0dcba812fd","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.0193947Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019403169Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.019745426Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"task":"crutest"} 01/30 Write block 293 len 32768 data: 4 4 5 5 7 10 9 8 02/30 Read block 283 len 4096 03/30 Read block 19 len 40960 04/30 Read block 300 len 28672 05/30 Read block 523 len 36864 06/30 Write block 235 len 20480 data: 6 6 4 4 5 07/30 Flush 08/30 Write block 365 len 12288 data: 9 10 7 09/30 Read block 177 len 40960 10/30 Write block 579 len 36864 data: 11 11 8 8 7 6 8 7 8 11/30 Read block 559 len 24576 12/30 Write block 182 len 8192 data: 3 5 13/30 Read block 11 len 40960 14/30 Write block 321 len 36864 data: 10 10 10 9 6 6 7 9 12 15/30 Read block 2 len 32768 16/30 Read block 379 len 8192 17/30 Read block 412 len 8192 18/30 Write block 394 len 36864 data: 10 11 11 11 11 12 10 10 10 19/30 Read block 65 len 12288 20/30 Flush 21/30 Write block 453 len 24576 data: 4 6 6 6 9 11 22/30 Read block 475 len 8192 23/30 Read block 529 len 32768 24/30 Read block 326 len 12288 25/30 Write block 318 len 32768 data: 14 13 12 11 11 11 10 7 26/30 Write block 309 len 20480 data: 10 10 13 15 13 27/30 Write block 416 len 40960 data: 12 10 9 9 7 6 6 5 4 6 28/30 Read block 344 len 40960 29/30 Read block 245 len 12288 30/30 Write block 442 len 4096 data: 12 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Write 5 Done Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Write 9 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 10 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 2 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 24 1 1 26 Done 6 29 29 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:02.02909007Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:02.029361222Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1493,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 159 159 159 F F F 1 020-039 87 87 87 159 159 159 F F F 2 040-059 88 88 88 160 160 160 F F F 3 060-079 87 87 87 159 159 159 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 88 88 88 160 160 160 F F F 6 120-139 84 84 84 154 154 154 F F F 7 140-159 87 87 87 159 159 159 F F F 8 160-179 84 84 84 154 154 154 F F F 9 180-199 89 90 90 161 163 163 F F F <--- 10 200-219 82 82 82 153 153 153 F F F 11 220-239 84 90 90 155 162 162 T F F <--- 12 240-259 88 88 88 160 160 160 F F F 13 260-279 88 88 88 160 160 160 F F F 14 280-299 88 90 90 160 162 162 T F F <--- 15 300-319 88 90 90 160 162 162 T T T <--- 16 320-339 87 90 90 159 163 163 F T T <--- 17 340-359 85 85 85 156 156 156 F F F 18 360-379 88 90 90 160 163 163 F F F <--- 19 380-399 88 90 90 160 163 163 F F F <--- 20 400-419 88 90 90 160 163 163 F T T <--- 21 420-439 89 89 89 161 161 161 F T T <--- 22 440-459 88 88 88 160 160 160 F T T <--- 23 460-479 86 86 86 157 157 157 F F F 24 480-499 85 85 85 156 156 156 F F F 25 500-519 87 87 87 159 159 159 F F F 26 520-539 87 87 87 159 159 159 F F F 27 540-559 87 87 87 159 159 159 F F F 28 560-579 87 90 90 159 163 163 F F F <--- 29 580-599 76 90 90 135 163 163 F F F <--- Max gen: 90, Max flush: 163 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 159 159 159 F F F 1 020-039 87 87 87 159 159 159 F F F 2 040-059 88 88 88 160 160 160 F F F 3 060-079 87 87 87 159 159 159 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 88 88 88 160 160 160 F F F 6 120-139 84 84 84 154 154 154 F F F 7 140-159 87 87 87 159 159 159 F F F 8 160-179 84 84 84 154 154 154 F F F 9 180-199 90 90 90 163 163 163 F F F 10 200-219 82 82 82 153 153 153 F F F 11 220-239 90 90 90 162 162 162 F F F 12 240-259 88 88 88 160 160 160 F F F 13 260-279 88 88 88 160 160 160 F F F 14 280-299 90 90 90 162 162 162 F F F 15 300-319 91 91 91 164 164 164 F F F 16 320-339 91 91 91 164 164 164 F F F 17 340-359 85 85 85 156 156 156 F F F 18 360-379 90 90 90 163 163 163 F F F 19 380-399 90 90 90 163 163 163 F F F 20 400-419 91 91 91 164 164 164 F F F 21 420-439 91 91 91 164 164 164 F F F 22 440-459 91 91 91 164 164 164 F F F 23 460-479 86 86 86 157 157 157 F F F 24 480-499 85 85 85 156 156 156 F F F 25 500-519 87 87 87 159 159 159 F F F 26 520-539 87 87 87 159 159 159 F F F 27 540-559 87 87 87 159 159 159 F F F 28 560-579 90 90 90 163 163 163 F F F 29 580-599 90 90 90 163 163 163 F F F Max gen: 91, Max flush: 164 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.277505872Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:06.284647456Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.284713552Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.291470102Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.291741464Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.291767272Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.291776291Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.291788141Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501} {"msg":"Crucible 8b04842b-09b1-4ad2-a9b7-8f92f34d809a has session id: 3bdd48d8-ce02-4b43-8b27-04dfb84283e3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292143726Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"Upstairs opts: Upstairs UUID: 8b04842b-09b1-4ad2-a9b7-8f92f34d809a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292176814Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"Crucible stats registered with UUID: 8b04842b-09b1-4ad2-a9b7-8f92f34d809a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292188373Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:06.292197953Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292336083Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292352982Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292363541Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292545809Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292563608Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501} {"msg":"8b04842b-09b1-4ad2-a9b7-8f92f34d809a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292581457Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292590766Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"client":"0","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292600805Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"client":"1","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292610825Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"client":"2","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292620824Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292631103Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292641513Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:49088)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292724067Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"io task","client":"1","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:41372)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292750965Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"io task","client":"0","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"ds_connection connected from Ok(127.0.0.1:65341)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.292762624Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"io task","client":"2","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293421989Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"client":"1","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293443898Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"client":"2","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293582868Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"client":"0","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293777895Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293787895Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"[0]R flush_numbers[0..12]: [159, 159, 160, 159, 159, 160, 154, 159, 154, 163, 153, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293876189Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"downstairs","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"[0]R generation[0..12]: [87, 87, 88, 87, 87, 88, 84, 87, 84, 90, 82, 90]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293887608Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"downstairs","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293898977Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"downstairs","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"[1]R flush_numbers[0..12]: [159, 159, 160, 159, 159, 160, 154, 159, 154, 163, 153, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293909816Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"downstairs","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"[1]R generation[0..12]: [87, 87, 88, 87, 87, 88, 84, 87, 84, 90, 82, 90]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293920406Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"downstairs","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293931745Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"downstairs","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"[2]R flush_numbers[0..12]: [159, 159, 160, 159, 159, 160, 154, 159, 154, 163, 153, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293945164Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"downstairs","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"[2]R generation[0..12]: [87, 87, 88, 87, 87, 88, 84, 87, 84, 90, 82, 90]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293955723Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"downstairs","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293966932Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"downstairs","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293976752Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"downstairs","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.293986671Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"downstairs","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.29399635Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"downstairs","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.2940066Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"":"downstairs","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.294017299Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.294026378Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"client":"0","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.294036658Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"client":"1","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.294046897Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"client":"2","":"downstairs","session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.294057226Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.294066766Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"8b04842b-09b1-4ad2-a9b7-8f92f34d809a is now active with session: 3bdd48d8-ce02-4b43-8b27-04dfb84283e3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.294077035Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.294086384Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.294094364Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"session_id":"3bdd48d8-ce02-4b43-8b27-04dfb84283e3"} {"msg":"Activated sub_volume 8b04842b-09b1-4ad2-a9b7-8f92f34d809a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.294105273Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.294113672Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.294416102Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"task":"crutest"} 01/30 Write block 179 len 20480 data: 4 3 3 4 6 02/30 Flush 03/30 Write block 318 len 16384 data: 15 14 13 12 04/30 Write block 389 len 24576 data: 4 6 7 6 7 11 05/30 Read block 399 len 32768 06/30 Read block 359 len 28672 07/30 Read block 495 len 40960 08/30 Flush 09/30 Write block 519 len 28672 data: 6 7 7 8 7 7 9 10/30 Write block 8 len 8192 data: 7 7 11/30 Write block 254 len 12288 data: 4 5 5 12/30 Write block 591 len 32768 data: 5 3 3 3 3 3 3 3 13/30 Write block 193 len 20480 data: 8 8 9 6 5 14/30 Write block 306 len 8192 data: 11 12 15/30 Flush 16/30 Write block 240 len 12288 data: 6 7 9 17/30 Flush 18/30 Write block 515 len 4096 data: 5 19/30 Read block 192 len 28672 20/30 Flush 21/30 Write block 144 len 24576 data: 14 13 13 13 12 12 22/30 Write block 565 len 36864 data: 9 7 9 9 10 11 10 11 12 23/30 Read block 542 len 28672 24/30 Write block 240 len 40960 data: 7 8 10 9 9 8 11 8 5 5 25/30 Read block 564 len 20480 26/30 Write block 22 len 12288 data: 6 4 3 27/30 Write block 253 len 28672 data: 2 5 6 6 7 8 7 28/30 Write block 58 len 16384 data: 8 7 6 7 29/30 Read block 151 len 36864 30/30 Read block 380 len 8192 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Sent Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 4 Sent Done Done false 1003 Acked Write 6 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Read 7 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 3 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Write 3 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Read 9 Sent Done Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 0 31 Done 0 29 30 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:06.302416857Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:06.302701488Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1501,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 92 92 159 167 167 F F F <--- 1 020-039 87 87 87 159 159 159 F T T <--- 2 040-059 88 88 88 160 160 160 F T T <--- 3 060-079 87 87 87 159 159 159 F T T <--- 4 080-099 87 87 87 159 159 159 F F F 5 100-119 88 88 88 160 160 160 F F F 6 120-139 84 84 84 154 154 154 F F F 7 140-159 87 87 87 159 159 159 F T T <--- 8 160-179 84 92 92 154 165 165 F F F <--- 9 180-199 90 92 92 163 167 167 F F F <--- 10 200-219 82 82 82 153 153 153 F F F 11 220-239 90 90 90 162 162 162 F F F 12 240-259 88 92 92 160 168 168 F T T <--- 13 260-279 88 88 88 160 160 160 F F F 14 280-299 90 90 90 162 162 162 F F F 15 300-319 91 92 92 164 167 167 F F F <--- 16 320-339 91 92 92 164 166 166 F F F <--- 17 340-359 85 85 85 156 156 156 F F F 18 360-379 90 90 90 163 163 163 F F F 19 380-399 90 92 92 163 166 166 F F F <--- 20 400-419 91 91 91 164 164 164 F F F 21 420-439 91 91 91 164 164 164 F F F 22 440-459 91 91 91 164 164 164 F F F 23 460-479 86 86 86 157 157 157 F F F 24 480-499 85 85 85 156 156 156 F F F 25 500-519 87 92 92 159 169 169 F F F <--- 26 520-539 87 92 92 159 167 167 F F F <--- 27 540-559 87 87 87 159 159 159 F F F 28 560-579 90 90 90 163 163 163 F T T <--- 29 580-599 90 92 92 163 167 167 F F F <--- Max gen: 92, Max flush: 169 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 167 167 167 F F F 1 020-039 93 93 93 170 170 170 F F F 2 040-059 93 93 93 170 170 170 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 88 88 88 160 160 160 F F F 6 120-139 84 84 84 154 154 154 F F F 7 140-159 93 93 93 170 170 170 F F F 8 160-179 92 92 92 165 165 165 F F F 9 180-199 92 92 92 167 167 167 F F F 10 200-219 82 82 82 153 153 153 F F F 11 220-239 90 90 90 162 162 162 F F F 12 240-259 93 93 93 170 170 170 F F F 13 260-279 88 88 88 160 160 160 F F F 14 280-299 90 90 90 162 162 162 F F F 15 300-319 92 92 92 167 167 167 F F F 16 320-339 92 92 92 166 166 166 F F F 17 340-359 85 85 85 156 156 156 F F F 18 360-379 90 90 90 163 163 163 F F F 19 380-399 92 92 92 166 166 166 F F F 20 400-419 91 91 91 164 164 164 F F F 21 420-439 91 91 91 164 164 164 F F F 22 440-459 91 91 91 164 164 164 F F F 23 460-479 86 86 86 157 157 157 F F F 24 480-499 85 85 85 156 156 156 F F F 25 500-519 92 92 92 169 169 169 F F F 26 520-539 92 92 92 167 167 167 F F F 27 540-559 87 87 87 159 159 159 F F F 28 560-579 93 93 93 170 170 170 F F F 29 580-599 92 92 92 167 167 167 F F F Max gen: 93, Max flush: 170 Begin loop 47 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.556194955Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:10.56332205Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.563379316Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.570148405Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.570409687Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.570434726Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.570443965Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.570455964Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509} {"msg":"Crucible 314e0d1d-6e92-494a-bad6-1a5e11e258e1 has session id: 1e1ec3aa-45be-45a7-83fd-929d46f9aa04","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.5708127Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"Upstairs opts: Upstairs UUID: 314e0d1d-6e92-494a-bad6-1a5e11e258e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.570837338Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"Crucible stats registered with UUID: 314e0d1d-6e92-494a-bad6-1a5e11e258e1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.570847738Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:10.570862197Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.571031475Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"io task","client":"1","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.571062813Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"io task","client":"0","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.571074092Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"io task","client":"2","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.571168686Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.571186365Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509} {"msg":"314e0d1d-6e92-494a-bad6-1a5e11e258e1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.571212543Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.571222322Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"client":"0","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.571232472Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"client":"1","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.571242671Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"client":"2","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.57125254Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"io task","client":"2","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.571263219Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"io task","client":"0","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.571273739Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"io task","client":"1","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"ds_connection connected from Ok(127.0.0.1:42939)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.571371832Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"io task","client":"1","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"ds_connection connected from Ok(127.0.0.1:48735)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.5713987Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"io task","client":"0","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"ds_connection connected from Ok(127.0.0.1:61118)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.571415039Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"io task","client":"2","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572074244Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"client":"2","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.57214346Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"client":"1","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572155969Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"client":"0","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572395552Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572421041Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"[0]R flush_numbers[0..12]: [167, 170, 170, 170, 159, 160, 154, 170, 165, 167, 153, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572482796Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"downstairs","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"[0]R generation[0..12]: [92, 93, 93, 93, 87, 88, 84, 93, 92, 92, 82, 90]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572494176Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"downstairs","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572505555Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"downstairs","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"[1]R flush_numbers[0..12]: [167, 170, 170, 170, 159, 160, 154, 170, 165, 167, 153, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572516604Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"downstairs","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"[1]R generation[0..12]: [92, 93, 93, 93, 87, 88, 84, 93, 92, 92, 82, 90]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572527283Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"downstairs","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572538523Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"downstairs","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"[2]R flush_numbers[0..12]: [167, 170, 170, 170, 159, 160, 154, 170, 165, 167, 153, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572549502Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"downstairs","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"[2]R generation[0..12]: [92, 93, 93, 93, 87, 88, 84, 93, 92, 92, 82, 90]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572560271Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"downstairs","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.57257149Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"downstairs","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.57258262Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"downstairs","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572592829Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"downstairs","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572602498Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"downstairs","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572612358Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"":"downstairs","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572622657Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572631866Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"client":"0","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572642206Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"client":"1","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572652385Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"client":"2","":"downstairs","session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572662764Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572672344Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"314e0d1d-6e92-494a-bad6-1a5e11e258e1 is now active with session: 1e1ec3aa-45be-45a7-83fd-929d46f9aa04","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572682623Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572692102Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572699982Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"session_id":"1e1ec3aa-45be-45a7-83fd-929d46f9aa04"} {"msg":"Activated sub_volume 314e0d1d-6e92-494a-bad6-1a5e11e258e1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.572709441Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.57271795Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.573029289Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"task":"crutest"} 01/30 Write block 426 len 36864 data: 7 8 8 6 7 9 9 9 9 02/30 Read block 362 len 32768 03/30 Read block 457 len 36864 04/30 Write block 381 len 20480 data: 9 10 10 7 8 05/30 Read block 261 len 24576 06/30 Write block 177 len 32768 data: 4 4 5 4 4 5 7 6 07/30 Flush 08/30 Write block 476 len 36864 data: 5 5 7 6 6 5 6 7 8 09/30 Write block 451 len 32768 data: 5 5 5 7 7 7 10 12 10/30 Read block 64 len 20480 11/30 Write block 405 len 28672 data: 8 8 8 8 6 6 8 12/30 Write block 24 len 36864 data: 4 4 4 5 6 6 9 11 8 13/30 Read block 81 len 12288 14/30 Read block 28 len 32768 15/30 Read block 53 len 8192 16/30 Read block 299 len 16384 17/30 Write block 420 len 40960 data: 8 7 7 6 5 7 8 9 9 7 18/30 Read block 436 len 12288 19/30 Write block 194 len 28672 data: 9 10 7 6 7 9 9 20/30 Read block 296 len 32768 21/30 Write block 251 len 40960 data: 3 2 3 6 7 7 8 9 8 7 22/30 Write block 234 len 24576 data: 5 7 7 5 5 6 23/30 Read block 354 len 24576 24/30 Read block 573 len 8192 25/30 Write block 365 len 12288 data: 10 11 8 26/30 Read block 436 len 4096 27/30 Write block 258 len 16384 data: 10 9 8 7 28/30 Read block 376 len 20480 29/30 Write block 438 len 32768 data: 10 10 11 11 13 9 7 6 30/30 Write block 181 len 40960 data: 5 6 8 7 8 6 6 8 7 7 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 9 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Write 9 Sent Done Done false 1012 Acked Read 3 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Read 4 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 3 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Read 5 Sent Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 20 2 2 24 Done 3 21 21 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1006)) Some(JobId(1006)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:10.582319727Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:10.582604388Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1509,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 167 167 167 F F F 1 020-039 93 93 93 170 170 170 F T T <--- 2 040-059 93 93 93 170 170 170 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 88 88 88 160 160 160 F F F 6 120-139 84 84 84 154 154 154 F F F 7 140-159 93 93 93 170 170 170 F F F 8 160-179 94 94 94 171 171 171 F F F 9 180-199 94 94 94 171 171 171 F T T <--- 10 200-219 82 82 82 153 153 153 F T T <--- 11 220-239 90 90 90 162 162 162 F T T <--- 12 240-259 93 93 93 170 170 170 F T T <--- 13 260-279 88 88 88 160 160 160 F T T <--- 14 280-299 90 90 90 162 162 162 F F F 15 300-319 92 92 92 167 167 167 F F F 16 320-339 92 92 92 166 166 166 F F F 17 340-359 85 85 85 156 156 156 F F F 18 360-379 90 90 90 163 163 163 F T T <--- 19 380-399 94 94 94 171 171 171 F F F 20 400-419 91 91 91 164 164 164 F T T <--- 21 420-439 94 94 94 171 171 171 F T T <--- 22 440-459 91 91 91 164 164 164 T T T 23 460-479 86 86 86 157 157 157 T T T 24 480-499 85 85 85 156 156 156 T T T 25 500-519 92 92 92 169 169 169 F F F 26 520-539 92 92 92 167 167 167 F F F 27 540-559 87 87 87 159 159 159 F F F 28 560-579 93 93 93 170 170 170 F F F 29 580-599 92 92 92 167 167 167 F F F Max gen: 94, Max flush: 171 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 47 Verifying data now Files /tmp/test_repair/test_repair_verify.data and /tmp/test_repair/test_repair_verify.data.last differ 443,446c443,447 < 10, < 10, < 12, < 8, --- > 11, > 11, > 13, > 9, > 7, 448d448 < 5, diff found after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 167 167 167 F F F 1 020-039 95 95 95 172 172 172 F F F 2 040-059 93 93 93 170 170 170 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 88 88 88 160 160 160 F F F 6 120-139 84 84 84 154 154 154 F F F 7 140-159 93 93 93 170 170 170 F F F 8 160-179 94 94 94 171 171 171 F F F 9 180-199 95 95 95 172 172 172 F F F 10 200-219 95 95 95 172 172 172 F F F 11 220-239 95 95 95 172 172 172 F F F 12 240-259 95 95 95 172 172 172 F F F 13 260-279 95 95 95 172 172 172 F F F 14 280-299 90 90 90 162 162 162 F F F 15 300-319 92 92 92 167 167 167 F F F 16 320-339 92 92 92 166 166 166 F F F 17 340-359 85 85 85 156 156 156 F F F 18 360-379 95 95 95 172 172 172 F F F 19 380-399 94 94 94 171 171 171 F F F 20 400-419 95 95 95 172 172 172 F F F 21 420-439 95 95 95 172 172 172 F F F 22 440-459 95 95 95 172 172 172 F F F 23 460-479 95 95 95 172 172 172 F F F 24 480-499 95 95 95 172 172 172 F F F 25 500-519 92 92 92 169 169 169 F F F 26 520-539 92 92 92 167 167 167 F F F 27 540-559 87 87 87 159 159 159 F F F 28 560-579 93 93 93 170 170 170 F F F 29 580-599 92 92 92 167 167 167 F F F Max gen: 95, Max flush: 172 Begin loop 48 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.840054915Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.848364Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.848643571Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.848673979Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.848682758Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.848694717Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520} {"msg":"Crucible b1291266-6a9c-40b4-bb90-a80f501351ea has session id: 80fc29d1-cd79-4ceb-8969-cf34b004dd8a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849009646Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"Upstairs opts: Upstairs UUID: b1291266-6a9c-40b4-bb90-a80f501351ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849033714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"Crucible stats registered with UUID: b1291266-6a9c-40b4-bb90-a80f501351ea","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849044184Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:14.849058003Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849219702Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.84923668Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.84924752Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849357532Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849374901Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520} {"msg":"b1291266-6a9c-40b4-bb90-a80f501351ea active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849401579Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849411229Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"client":"0","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849421228Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"client":"1","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849431137Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"client":"2","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849441147Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849454116Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849465485Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48454)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849513342Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48396)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.84954105Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:52610)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.849552969Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850133779Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"client":"0","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850155618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"client":"1","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850247652Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"client":"2","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850472616Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850496625Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"[0]R flush_numbers[0..12]: [167, 172, 170, 170, 159, 160, 154, 170, 171, 172, 172, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850507794Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"downstairs","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"[0]R generation[0..12]: [92, 95, 93, 93, 87, 88, 84, 93, 94, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850521993Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"downstairs","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850533482Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"downstairs","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"[1]R flush_numbers[0..12]: [167, 172, 170, 170, 159, 160, 154, 170, 171, 172, 172, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850544401Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"downstairs","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"[1]R generation[0..12]: [92, 95, 93, 93, 87, 88, 84, 93, 94, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850555161Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"downstairs","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.85056648Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"downstairs","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"[2]R flush_numbers[0..12]: [167, 172, 170, 170, 159, 160, 154, 170, 171, 172, 172, 172]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850577679Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"downstairs","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"[2]R generation[0..12]: [92, 95, 93, 93, 87, 88, 84, 93, 94, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850588478Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"downstairs","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850599688Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"downstairs","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850609437Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"downstairs","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850619486Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"downstairs","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850629156Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"downstairs","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850638955Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"":"downstairs","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850648954Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850657934Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"client":"0","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850669403Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"client":"1","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850679412Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"client":"2","":"downstairs","session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850689582Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850699001Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"b1291266-6a9c-40b4-bb90-a80f501351ea is now active with session: 80fc29d1-cd79-4ceb-8969-cf34b004dd8a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.85070924Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.85071831Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"session_id":"80fc29d1-cd79-4ceb-8969-cf34b004dd8a"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850728009Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520} {"msg":"Activated sub_volume b1291266-6a9c-40b4-bb90-a80f501351ea","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850736668Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.850745288Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.851067446Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"task":"crutest"} 01/30 Write block 535 len 32768 data: 8 7 7 7 7 7 5 8 02/30 Write block 238 len 24576 data: 6 7 8 9 11 10 03/30 Flush 04/30 Write block 465 len 16384 data: 14 14 10 10 05/30 Read block 216 len 28672 06/30 Read block 374 len 28672 07/30 Write block 149 len 8192 data: 13 13 08/30 Write block 397 len 32768 data: 12 12 13 11 11 11 9 8 09/30 Read block 205 len 28672 10/30 Flush 11/30 Write block 201 len 36864 data: 8 7 7 10 11 11 10 13 10 12/30 Read block 415 len 32768 13/30 Write block 107 len 40960 data: 8 10 9 9 9 7 7 8 7 6 14/30 Read block 517 len 40960 15/30 Write block 32 len 36864 data: 9 11 9 9 9 7 8 10 10 16/30 Read block 358 len 8192 17/30 Flush 18/30 Write block 224 len 20480 data: 9 7 9 10 12 19/30 Write block 241 len 20480 data: 10 12 11 10 9 20/30 Read block 385 len 20480 21/30 Write block 530 len 40960 data: 7 8 9 8 7 9 8 8 8 8 22/30 Read block 370 len 8192 23/30 Write block 420 len 20480 data: 9 8 8 7 6 24/30 Read block 357 len 12288 25/30 Read block 85 len 4096 26/30 Write block 287 len 4096 data: 5 27/30 Write block 538 len 16384 data: 9 9 8 6 28/30 Write block 534 len 28672 data: 8 10 9 9 10 10 9 29/30 Write block 589 len 24576 data: 3 3 6 4 4 4 30/30 Read block 559 len 24576 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 4 Done Done Done false 1004 Acked Read 7 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Write 8 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Read 8 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Write 5 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 6 Done Done Sent false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 23 23 Done 27 27 4 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:14.860090612Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:14.860371183Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1520,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 167 167 167 F F F 1 020-039 96 96 95 175 175 172 F F F <--- 2 040-059 96 96 93 175 175 170 F F F <--- 3 060-079 93 93 93 170 170 170 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 96 96 88 175 175 160 F F F <--- 6 120-139 84 84 84 154 154 154 F F F 7 140-159 96 96 93 174 174 170 F F T <--- 8 160-179 94 94 94 171 171 171 F F F 9 180-199 95 95 95 172 172 172 F F F 10 200-219 96 96 95 175 175 172 F F F <--- 11 220-239 96 96 96 173 173 173 T T F <--- 12 240-259 96 96 96 173 173 173 T T F <--- 13 260-279 95 95 95 172 172 172 F F F 14 280-299 90 90 90 162 162 162 T T F <--- 15 300-319 92 92 92 167 167 167 F F F 16 320-339 92 92 92 166 166 166 F F F 17 340-359 85 85 85 156 156 156 F F F 18 360-379 95 95 95 172 172 172 F F F 19 380-399 96 96 94 174 174 171 F F F <--- 20 400-419 96 96 95 174 174 172 F F F <--- 21 420-439 95 95 95 172 172 172 T T F <--- 22 440-459 95 95 95 172 172 172 F F F 23 460-479 96 96 95 174 174 172 F F T <--- 24 480-499 95 95 95 172 172 172 F F F 25 500-519 92 92 92 169 169 169 F F F 26 520-539 96 96 96 173 173 173 T T F <--- 27 540-559 96 96 96 173 173 173 T T F <--- 28 560-579 93 93 93 170 170 170 F F F 29 580-599 92 92 92 167 167 167 T T F <--- Max gen: 96, Max flush: 175 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 167 167 167 F F F 1 020-039 96 96 96 175 175 175 F F F 2 040-059 96 96 96 175 175 175 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 96 96 96 175 175 175 F F F 6 120-139 84 84 84 154 154 154 F F F 7 140-159 96 96 96 174 174 174 F F F 8 160-179 94 94 94 171 171 171 F F F 9 180-199 95 95 95 172 172 172 F F F 10 200-219 96 96 96 175 175 175 F F F 11 220-239 97 97 97 176 176 176 F F F 12 240-259 97 97 97 176 176 176 F F F 13 260-279 95 95 95 172 172 172 F F F 14 280-299 97 97 97 176 176 176 F F F 15 300-319 92 92 92 167 167 167 F F F 16 320-339 92 92 92 166 166 166 F F F 17 340-359 85 85 85 156 156 156 F F F 18 360-379 95 95 95 172 172 172 F F F 19 380-399 96 96 96 174 174 174 F F F 20 400-419 96 96 96 174 174 174 F F F 21 420-439 97 97 97 176 176 176 F F F 22 440-459 95 95 95 172 172 172 F F F 23 460-479 96 96 96 174 174 174 F F F 24 480-499 95 95 95 172 172 172 F F F 25 500-519 92 92 92 169 169 169 F F F 26 520-539 97 97 97 176 176 176 F F F 27 540-559 97 97 97 176 176 176 F F F 28 560-579 93 93 93 170 170 170 F F F 29 580-599 97 97 97 176 176 176 F F F Max gen: 97, Max flush: 176 Begin loop 49 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.110016751Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.118183365Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.118468836Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.118495594Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.118504714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.118516713Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528} {"msg":"Crucible f420864c-e9ea-49b5-abbb-e2ca4c2432ac has session id: 2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.118869109Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"Upstairs opts: Upstairs UUID: f420864c-e9ea-49b5-abbb-e2ca4c2432ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.118889497Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"Crucible stats registered with UUID: f420864c-e9ea-49b5-abbb-e2ca4c2432ac","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.118900157Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:19.118912126Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.119086744Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.119119042Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.119130311Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.119224665Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.119249463Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528} {"msg":"f420864c-e9ea-49b5-abbb-e2ca4c2432ac active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.119276401Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.11928615Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"client":"0","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.11929667Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"client":"1","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.119306689Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"client":"2","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.119316758Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.119327148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.119355766Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"ds_connection connected from Ok(127.0.0.1:50572)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.119398053Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"ds_connection connected from Ok(127.0.0.1:45656)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.119429501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"ds_connection connected from Ok(127.0.0.1:53352)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.119454589Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120193529Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"client":"2","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120246755Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"client":"1","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120311281Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"client":"0","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120529456Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120550414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"[0]R flush_numbers[0..12]: [167, 175, 175, 170, 159, 175, 154, 174, 171, 172, 175, 176]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120606901Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"downstairs","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"[0]R generation[0..12]: [92, 96, 96, 93, 87, 96, 84, 96, 94, 95, 96, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.12061877Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"downstairs","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120630399Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"downstairs","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"[1]R flush_numbers[0..12]: [167, 175, 175, 170, 159, 175, 154, 174, 171, 172, 175, 176]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120641648Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"downstairs","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"[1]R generation[0..12]: [92, 96, 96, 93, 87, 96, 84, 96, 94, 95, 96, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120652457Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"downstairs","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120666607Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"downstairs","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"[2]R flush_numbers[0..12]: [167, 175, 175, 170, 159, 175, 154, 174, 171, 172, 175, 176]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120677696Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"downstairs","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"[2]R generation[0..12]: [92, 96, 96, 93, 87, 96, 84, 96, 94, 95, 96, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120688355Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"downstairs","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120699644Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"downstairs","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120709554Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"downstairs","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120719763Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"downstairs","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120729572Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"downstairs","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120739502Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"":"downstairs","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120749541Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.12075879Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"client":"0","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.12076894Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"client":"1","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120779109Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"client":"2","":"downstairs","session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120789388Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120798928Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"f420864c-e9ea-49b5-abbb-e2ca4c2432ac is now active with session: 2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120809387Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120818756Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120827846Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"session_id":"2ab4bd8c-8d85-425f-9d6e-2a13ff7fec35"} {"msg":"Activated sub_volume f420864c-e9ea-49b5-abbb-e2ca4c2432ac","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120837345Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.120845954Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.121114236Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"task":"crutest"} 01/30 Write block 156 len 40960 data: 6 5 4 7 7 6 6 6 8 9 02/30 Write block 300 len 12288 data: 9 8 8 03/30 Read block 194 len 40960 04/30 Write block 377 len 28672 data: 11 14 12 10 10 11 11 05/30 Write block 161 len 24576 data: 7 7 7 9 10 10 06/30 Read block 172 len 16384 07/30 Flush 08/30 Write block 7 len 12288 data: 9 8 8 09/30 Write block 525 len 16384 data: 10 8 7 7 10/30 Flush 11/30 Write block 524 len 32768 data: 8 11 9 8 8 8 8 9 12/30 Write block 306 len 4096 data: 12 13/30 Write block 318 len 8192 data: 16 15 14/30 Write block 332 len 12288 data: 13 13 11 15/30 Read block 271 len 16384 16/30 Flush 17/30 Write block 520 len 20480 data: 8 8 9 8 9 18/30 Read block 37 len 4096 19/30 Flush 20/30 Write block 245 len 32768 data: 10 12 9 6 6 4 4 3 21/30 Read block 157 len 8192 22/30 Write block 303 len 32768 data: 8 10 9 13 13 9 11 11 23/30 Read block 224 len 12288 24/30 Write block 466 len 32768 data: 15 11 11 11 12 10 8 7 25/30 Read block 111 len 36864 26/30 Write block 513 len 8192 data: 6 6 27/30 Write block 527 len 16384 data: 9 9 9 9 28/30 Read block 213 len 24576 29/30 Write block 573 len 36864 data: 13 11 14 13 12 12 12 12 9 30/30 Write block 122 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Write 4 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 1 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 26 2 30 Done 28 4 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:19.129369714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:19.129686793Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1528,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 92 98 178 167 178 F F F <--- 1 020-039 96 96 96 175 175 175 F F F 2 040-059 96 96 96 175 175 175 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 96 96 96 175 175 175 F F F 6 120-139 84 84 84 154 154 154 T F T <--- 7 140-159 98 96 98 177 174 177 F T F <--- 8 160-179 98 94 98 177 171 177 F T F <--- 9 180-199 95 95 95 172 172 172 F F F 10 200-219 96 96 96 175 175 175 F F F 11 220-239 97 97 97 176 176 176 F F F 12 240-259 97 97 97 176 176 176 T F T <--- 13 260-279 95 95 95 172 172 172 F F F 14 280-299 97 97 97 176 176 176 F F F 15 300-319 98 92 98 179 167 179 T T T <--- 16 320-339 98 92 98 179 166 179 F F F <--- 17 340-359 85 85 85 156 156 156 F F F 18 360-379 98 95 98 177 172 177 F T F <--- 19 380-399 98 96 98 177 174 177 F T F <--- 20 400-419 96 96 96 174 174 174 F F F 21 420-439 97 97 97 176 176 176 F F F 22 440-459 95 95 95 172 172 172 F F F 23 460-479 96 96 96 174 174 174 T F T <--- 24 480-499 95 95 95 172 172 172 F F F 25 500-519 92 92 92 169 169 169 T F T <--- 26 520-539 98 97 98 180 176 180 T F T <--- 27 540-559 97 97 97 176 176 176 F F F 28 560-579 93 93 93 170 170 170 T F T <--- 29 580-599 97 97 97 176 176 176 T F T <--- Max gen: 98, Max flush: 180 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 178 178 178 F F F 1 020-039 96 96 96 175 175 175 F F F 2 040-059 96 96 96 175 175 175 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 96 96 96 175 175 175 F F F 6 120-139 99 99 99 181 181 181 F F F 7 140-159 98 98 98 177 177 177 F F F 8 160-179 98 98 98 177 177 177 F F F 9 180-199 95 95 95 172 172 172 F F F 10 200-219 96 96 96 175 175 175 F F F 11 220-239 97 97 97 176 176 176 F F F 12 240-259 99 99 99 181 181 181 F F F 13 260-279 95 95 95 172 172 172 F F F 14 280-299 97 97 97 176 176 176 F F F 15 300-319 99 99 99 181 181 181 F F F 16 320-339 98 98 98 179 179 179 F F F 17 340-359 85 85 85 156 156 156 F F F 18 360-379 98 98 98 177 177 177 F F F 19 380-399 98 98 98 177 177 177 F F F 20 400-419 96 96 96 174 174 174 F F F 21 420-439 97 97 97 176 176 176 F F F 22 440-459 95 95 95 172 172 172 F F F 23 460-479 99 99 99 181 181 181 F F F 24 480-499 95 95 95 172 172 172 F F F 25 500-519 99 99 99 181 181 181 F F F 26 520-539 99 99 99 181 181 181 F F F 27 540-559 97 97 97 176 176 176 F F F 28 560-579 99 99 99 181 181 181 F F F 29 580-599 99 99 99 181 181 181 F F F Max gen: 99, Max flush: 181 Begin loop 50 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.377070775Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.38523122Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.385510921Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.385537509Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.385546448Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.385558438Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536} {"msg":"Crucible 8f4582bd-60b2-4dea-8acb-b2d66243276b has session id: 306aaee2-3e4e-45a8-a601-5dceaaea6c87","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.38596418Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"Upstairs opts: Upstairs UUID: 8f4582bd-60b2-4dea-8acb-b2d66243276b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.385994628Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"Crucible stats registered with UUID: 8f4582bd-60b2-4dea-8acb-b2d66243276b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386006477Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:23.386016556Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386186945Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386204044Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386214623Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386323775Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386340824Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536} {"msg":"8f4582bd-60b2-4dea-8acb-b2d66243276b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386373092Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386382861Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"client":"0","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386393061Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"client":"1","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.38640299Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"client":"2","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386412949Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386423709Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386474375Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"ds_connection connected from Ok(127.0.0.1:40252)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386485604Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"ds_connection connected from Ok(127.0.0.1:53851)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386496414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"ds_connection connected from Ok(127.0.0.1:43192)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.386625955Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387214135Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"client":"2","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387228844Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"client":"0","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387306189Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"client":"1","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387549762Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387559721Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"[0]R flush_numbers[0..12]: [178, 175, 175, 170, 159, 175, 181, 177, 177, 172, 175, 176]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387607438Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"downstairs","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"[0]R generation[0..12]: [98, 96, 96, 93, 87, 96, 99, 98, 98, 95, 96, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387618817Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"downstairs","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387630327Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"downstairs","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"[1]R flush_numbers[0..12]: [178, 175, 175, 170, 159, 175, 181, 177, 177, 172, 175, 176]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387641296Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"downstairs","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"[1]R generation[0..12]: [98, 96, 96, 93, 87, 96, 99, 98, 98, 95, 96, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387651995Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"downstairs","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387663074Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"downstairs","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"[2]R flush_numbers[0..12]: [178, 175, 175, 170, 159, 175, 181, 177, 177, 172, 175, 176]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387673914Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"downstairs","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"[2]R generation[0..12]: [98, 96, 96, 93, 87, 96, 99, 98, 98, 95, 96, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387684643Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"downstairs","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387695982Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"downstairs","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387705781Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"downstairs","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387717441Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"downstairs","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"Next flush: 182","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.38772715Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"downstairs","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387737109Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"":"downstairs","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387747229Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387756258Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"client":"0","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387766137Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"client":"1","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387776027Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"client":"2","":"downstairs","session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387786206Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387795655Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"8f4582bd-60b2-4dea-8acb-b2d66243276b is now active with session: 306aaee2-3e4e-45a8-a601-5dceaaea6c87","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387805835Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387814924Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"session_id":"306aaee2-3e4e-45a8-a601-5dceaaea6c87"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387824233Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536} {"msg":"Activated sub_volume 8f4582bd-60b2-4dea-8acb-b2d66243276b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387832703Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.387841322Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.38817386Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"task":"crutest"} 01/30 Write block 292 len 12288 data: 4 5 5 02/30 Write block 438 len 20480 data: 11 11 11 11 13 03/30 Write block 197 len 20480 data: 7 8 10 10 9 04/30 Write block 535 len 20480 data: 11 10 10 11 11 05/30 Read block 189 len 32768 06/30 Read block 558 len 8192 07/30 Read block 380 len 12288 08/30 Write block 33 len 8192 data: 12 10 09/30 Flush 10/30 Write block 22 len 20480 data: 7 5 5 5 5 11/30 Write block 246 len 4096 data: 13 12/30 Read block 361 len 8192 13/30 Read block 122 len 8192 14/30 Write block 151 len 4096 data: 13 15/30 Write block 173 len 12288 data: 13 12 9 16/30 Read block 351 len 16384 17/30 Read block 504 len 36864 18/30 Flush 19/30 Write block 551 len 12288 data: 8 8 8 20/30 Read block 461 len 32768 21/30 Read block 401 len 12288 22/30 Read block 542 len 36864 23/30 Write block 541 len 12288 data: 7 9 7 24/30 Flush 25/30 Write block 524 len 12288 data: 10 12 10 26/30 Read block 196 len 40960 27/30 Write block 597 len 8192 data: 4 4 28/30 Read block 407 len 28672 29/30 Read block 306 len 32768 30/30 Read block 190 len 12288 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Sent Done false 1001 Acked Write 5 Done Sent Done false 1002 Acked Write 5 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Read 3 Done Sent Done false 1007 Acked Write 2 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Read 9 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:23.395746154Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:23.396044664Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1536,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 178 178 178 F F F 1 020-039 100 96 100 183 175 183 F F F <--- 2 040-059 96 96 96 175 175 175 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 96 96 96 175 175 175 F F F 6 120-139 99 99 99 181 181 181 F F F 7 140-159 100 98 100 183 177 183 F F F <--- 8 160-179 100 98 100 183 177 183 F F F <--- 9 180-199 100 95 100 182 172 182 F F F <--- 10 200-219 100 96 100 182 175 182 F F F <--- 11 220-239 97 97 97 176 176 176 F F F 12 240-259 100 99 100 183 181 183 F F F <--- 13 260-279 95 95 95 172 172 172 F F F 14 280-299 100 97 100 182 176 182 F F F <--- 15 300-319 99 99 99 181 181 181 F F F 16 320-339 98 98 98 179 179 179 F F F 17 340-359 85 85 85 156 156 156 F F F 18 360-379 98 98 98 177 177 177 F F F 19 380-399 98 98 98 177 177 177 F F F 20 400-419 96 96 96 174 174 174 F F F 21 420-439 100 97 100 182 176 182 F F F <--- 22 440-459 100 95 100 182 172 182 F F F <--- 23 460-479 99 99 99 181 181 181 F F F 24 480-499 95 95 95 172 172 172 F F F 25 500-519 99 99 99 181 181 181 F F F 26 520-539 100 99 100 182 181 182 T F T <--- 27 540-559 100 97 100 184 176 184 F F F <--- 28 560-579 99 99 99 181 181 181 F F F 29 580-599 99 99 99 181 181 181 T F T <--- Max gen: 100, Max flush: 184 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 178 178 178 F F F 1 020-039 100 100 100 183 183 183 F F F 2 040-059 96 96 96 175 175 175 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 96 96 96 175 175 175 F F F 6 120-139 99 99 99 181 181 181 F F F 7 140-159 100 100 100 183 183 183 F F F 8 160-179 100 100 100 183 183 183 F F F 9 180-199 100 100 100 182 182 182 F F F 10 200-219 100 100 100 182 182 182 F F F 11 220-239 97 97 97 176 176 176 F F F 12 240-259 100 100 100 183 183 183 F F F 13 260-279 95 95 95 172 172 172 F F F 14 280-299 100 100 100 182 182 182 F F F 15 300-319 99 99 99 181 181 181 F F F 16 320-339 98 98 98 179 179 179 F F F 17 340-359 85 85 85 156 156 156 F F F 18 360-379 98 98 98 177 177 177 F F F 19 380-399 98 98 98 177 177 177 F F F 20 400-419 96 96 96 174 174 174 F F F 21 420-439 100 100 100 182 182 182 F F F 22 440-459 100 100 100 182 182 182 F F F 23 460-479 99 99 99 181 181 181 F F F 24 480-499 95 95 95 172 172 172 F F F 25 500-519 99 99 99 181 181 181 F F F 26 520-539 101 101 101 185 185 185 F F F 27 540-559 100 100 100 184 184 184 F F F 28 560-579 99 99 99 181 181 181 F F F 29 580-599 101 101 101 185 185 185 F F F Max gen: 101, Max flush: 185 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.63940309Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.647567065Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.647851765Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.647880373Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.647889383Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.647901512Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544} {"msg":"Crucible b7220405-5d28-4f82-9089-a6aaf7b9119a has session id: 6f1f4932-8f04-465d-9e33-91ba2d55ea51","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648255918Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"Upstairs opts: Upstairs UUID: b7220405-5d28-4f82-9089-a6aaf7b9119a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648307024Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"Crucible stats registered with UUID: b7220405-5d28-4f82-9089-a6aaf7b9119a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648332933Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:27.648343222Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648510091Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648541738Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648553158Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648644122Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.64866148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544} {"msg":"b7220405-5d28-4f82-9089-a6aaf7b9119a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648685589Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648695298Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"client":"0","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648707877Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"client":"1","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648717946Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"client":"2","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648727716Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648738055Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648790902Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"ds_connection connected from Ok(127.0.0.1:36246)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648802051Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"ds_connection connected from Ok(127.0.0.1:41093)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.64881273Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"ds_connection connected from Ok(127.0.0.1:49343)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.648852247Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.649595627Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"client":"2","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.649616865Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"client":"0","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.649632734Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"client":"1","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.649918735Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.649952702Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"[0]R flush_numbers[0..12]: [178, 183, 175, 170, 159, 175, 181, 183, 183, 182, 182, 176]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650010149Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"downstairs","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"[0]R generation[0..12]: [98, 100, 96, 93, 87, 96, 99, 100, 100, 100, 100, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650021858Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"downstairs","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650035727Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"downstairs","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"[1]R flush_numbers[0..12]: [178, 183, 175, 170, 159, 175, 181, 183, 183, 182, 182, 176]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650046596Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"downstairs","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"[1]R generation[0..12]: [98, 100, 96, 93, 87, 96, 99, 100, 100, 100, 100, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650057335Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"downstairs","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650068495Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"downstairs","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"[2]R flush_numbers[0..12]: [178, 183, 175, 170, 159, 175, 181, 183, 183, 182, 182, 176]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650079374Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"downstairs","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"[2]R generation[0..12]: [98, 100, 96, 93, 87, 96, 99, 100, 100, 100, 100, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650090093Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"downstairs","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650101262Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"downstairs","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650111112Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"downstairs","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650121121Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"downstairs","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.65013102Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"downstairs","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.65014099Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"":"downstairs","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650151139Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650160348Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"client":"0","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650170258Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"client":"1","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650181567Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"client":"2","":"downstairs","session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650191666Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650201086Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"b7220405-5d28-4f82-9089-a6aaf7b9119a is now active with session: 6f1f4932-8f04-465d-9e33-91ba2d55ea51","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650211245Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650220664Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650228584Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"session_id":"6f1f4932-8f04-465d-9e33-91ba2d55ea51"} {"msg":"Activated sub_volume b7220405-5d28-4f82-9089-a6aaf7b9119a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650238033Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650246462Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.650596969Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"task":"crutest"} 01/30 Write block 388 len 20480 data: 3 5 7 8 7 02/30 Flush 03/30 Write block 343 len 8192 data: 9 10 04/30 Read block 74 len 40960 05/30 Write block 428 len 28672 data: 10 8 8 10 10 10 10 06/30 Read block 345 len 4096 07/30 Write block 347 len 8192 data: 5 4 08/30 Write block 589 len 40960 data: 4 4 7 5 5 5 4 4 5 5 09/30 Read block 374 len 28672 10/30 Write block 249 len 36864 data: 7 5 5 4 4 7 8 8 9 11/30 Write block 11 len 8192 data: 8 8 12/30 Read block 420 len 28672 13/30 Read block 359 len 12288 14/30 Write block 590 len 32768 data: 5 8 6 6 6 5 5 6 15/30 Read block 161 len 16384 16/30 Read block 475 len 40960 17/30 Write block 215 len 20480 data: 9 9 7 7 7 18/30 Write block 504 len 32768 data: 9 11 9 12 11 11 10 8 19/30 Flush 20/30 Write block 478 len 4096 data: 8 21/30 Read block 404 len 40960 22/30 Write block 136 len 24576 data: 8 9 10 11 9 10 23/30 Read block 470 len 24576 24/30 Flush 25/30 Flush 26/30 Write block 481 len 36864 data: 6 7 8 9 9 9 11 9 9 27/30 Read block 305 len 32768 28/30 Write block 532 len 32768 data: 10 9 9 12 11 11 12 12 29/30 Write block 158 len 24576 data: 5 8 8 8 8 8 30/30 Read block 526 len 8192 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 2 Done Sent Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Write 2 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Read 7 Done Sent Done false 1009 Acked Write 9 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Write 8 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 1 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Write 8 Done Sent Sent false 1028 Acked Write 6 Done Sent Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 3 32 Done 30 1 27 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:27.65984057Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:27.6601231Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1544,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 98 102 187 178 187 F F F <--- 1 020-039 100 100 100 183 183 183 F F F 2 040-059 96 96 96 175 175 175 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 96 96 96 175 175 175 F F F 6 120-139 102 99 102 188 181 188 F F F <--- 7 140-159 102 100 102 188 183 188 T F T <--- 8 160-179 100 100 100 183 183 183 T F T <--- 9 180-199 100 100 100 182 182 182 F F F 10 200-219 102 100 102 187 182 187 F F F <--- 11 220-239 97 97 97 176 176 176 F F F 12 240-259 102 100 102 187 183 187 F F F <--- 13 260-279 95 95 95 172 172 172 F F F 14 280-299 100 100 100 182 182 182 F F F 15 300-319 99 99 99 181 181 181 F F F 16 320-339 98 98 98 179 179 179 F F F 17 340-359 102 85 102 187 156 187 F F F <--- 18 360-379 98 98 98 177 177 177 F F F 19 380-399 102 98 102 186 177 186 F T F <--- 20 400-419 96 96 96 174 174 174 F F F 21 420-439 102 100 102 187 182 187 F F F <--- 22 440-459 100 100 100 182 182 182 F F F 23 460-479 102 99 102 188 181 188 F F F <--- 24 480-499 95 95 95 172 172 172 T F T <--- 25 500-519 102 99 102 187 181 187 F F F <--- 26 520-539 101 101 101 185 185 185 T F T <--- 27 540-559 100 100 100 184 184 184 F F F 28 560-579 99 99 99 181 181 181 F F F 29 580-599 102 101 102 187 185 187 F F F <--- Max gen: 102, Max flush: 188 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 187 187 187 F F F 1 020-039 100 100 100 183 183 183 F F F 2 040-059 96 96 96 175 175 175 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 87 87 87 159 159 159 F F F 5 100-119 96 96 96 175 175 175 F F F 6 120-139 102 102 102 188 188 188 F F F 7 140-159 103 103 103 189 189 189 F F F 8 160-179 103 103 103 189 189 189 F F F 9 180-199 100 100 100 182 182 182 F F F 10 200-219 102 102 102 187 187 187 F F F 11 220-239 97 97 97 176 176 176 F F F 12 240-259 102 102 102 187 187 187 F F F 13 260-279 95 95 95 172 172 172 F F F 14 280-299 100 100 100 182 182 182 F F F 15 300-319 99 99 99 181 181 181 F F F 16 320-339 98 98 98 179 179 179 F F F 17 340-359 102 102 102 187 187 187 F F F 18 360-379 98 98 98 177 177 177 F F F 19 380-399 102 102 102 186 186 186 F F F 20 400-419 96 96 96 174 174 174 F F F 21 420-439 102 102 102 187 187 187 F F F 22 440-459 100 100 100 182 182 182 F F F 23 460-479 102 102 102 188 188 188 F F F 24 480-499 103 103 103 189 189 189 F F F 25 500-519 102 102 102 187 187 187 F F F 26 520-539 103 103 103 189 189 189 F F F 27 540-559 100 100 100 184 184 184 F F F 28 560-579 99 99 99 181 181 181 F F F 29 580-599 102 102 102 187 187 187 F F F Max gen: 103, Max flush: 189 Begin loop 52 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.905725023Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.913947543Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.914232244Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.914259012Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.914268092Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.914279941Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552} {"msg":"Crucible c6082aac-6f25-4223-b8d4-b2a2bc330935 has session id: 11b242cf-eff5-41b8-a9f4-76e6cb455455","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.914639646Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"Upstairs opts: Upstairs UUID: c6082aac-6f25-4223-b8d4-b2a2bc330935, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.914672304Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"Crucible stats registered with UUID: c6082aac-6f25-4223-b8d4-b2a2bc330935","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.914683893Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:31.914693513Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.914858501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.914889409Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.914900679Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.914981233Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.914998912Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552} {"msg":"c6082aac-6f25-4223-b8d4-b2a2bc330935 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.915034109Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.915043969Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"client":"0","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.915054068Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"client":"1","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.915064117Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"client":"2","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.915073947Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.915084276Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.915138312Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"ds_connection connected from Ok(127.0.0.1:53710)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.91517532Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"ds_connection connected from Ok(127.0.0.1:59922)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.915215737Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"ds_connection connected from Ok(127.0.0.1:38735)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.915226906Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.915866813Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"client":"0","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.915900671Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"client":"1","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.91591281Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"client":"2","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916225528Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916237028Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"[0]R flush_numbers[0..12]: [187, 183, 175, 170, 159, 175, 188, 189, 189, 182, 187, 176]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916274155Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"downstairs","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"[0]R generation[0..12]: [102, 100, 96, 93, 87, 96, 102, 103, 103, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916285814Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"downstairs","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916297184Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"downstairs","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"[1]R flush_numbers[0..12]: [187, 183, 175, 170, 159, 175, 188, 189, 189, 182, 187, 176]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916308253Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"downstairs","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"[1]R generation[0..12]: [102, 100, 96, 93, 87, 96, 102, 103, 103, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916319102Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"downstairs","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916330311Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"downstairs","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"[2]R flush_numbers[0..12]: [187, 183, 175, 170, 159, 175, 188, 189, 189, 182, 187, 176]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.91634365Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"downstairs","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"[2]R generation[0..12]: [102, 100, 96, 93, 87, 96, 102, 103, 103, 100, 102, 97]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.91635441Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"downstairs","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916365569Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"downstairs","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916375328Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"downstairs","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916385428Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"downstairs","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916395017Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"downstairs","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916404656Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"":"downstairs","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916414656Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916423765Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"client":"0","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916433684Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"client":"1","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916443504Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"client":"2","":"downstairs","session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916453633Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916462962Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"c6082aac-6f25-4223-b8d4-b2a2bc330935 is now active with session: 11b242cf-eff5-41b8-a9f4-76e6cb455455","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916473132Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916482631Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.91649052Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"session_id":"11b242cf-eff5-41b8-a9f4-76e6cb455455"} {"msg":"Activated sub_volume c6082aac-6f25-4223-b8d4-b2a2bc330935","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.91650113Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916509549Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.916829057Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"task":"crutest"} 01/30 Write block 125 len 40960 data: 9 9 6 7 6 6 6 8 6 8 02/30 Read block 341 len 40960 03/30 Read block 190 len 8192 04/30 Write block 218 len 24576 data: 8 8 8 8 7 6 05/30 Read block 323 len 40960 06/30 Write block 385 len 12288 data: 9 7 4 07/30 Write block 107 len 4096 data: 9 08/30 Read block 11 len 28672 09/30 Read block 201 len 12288 10/30 Read block 461 len 24576 11/30 Read block 471 len 12288 12/30 Write block 81 len 12288 data: 8 6 5 13/30 Read block 582 len 40960 14/30 Read block 80 len 36864 15/30 Flush 16/30 Write block 533 len 12288 data: 10 10 13 17/30 Flush 18/30 Write block 563 len 20480 data: 9 9 10 8 10 19/30 Read block 376 len 12288 20/30 Read block 89 len 4096 21/30 Write block 396 len 12288 data: 12 13 13 22/30 Write block 541 len 40960 data: 8 10 8 9 9 9 9 10 10 9 23/30 Read block 181 len 36864 24/30 Read block 152 len 28672 25/30 Read block 119 len 24576 26/30 Write block 256 len 40960 data: 9 10 11 10 9 8 7 8 8 6 27/30 Write block 161 len 28672 data: 9 9 9 10 11 11 13 28/30 Read block 173 len 16384 29/30 Write block 431 len 20480 data: 11 11 11 11 11 30/30 Write block 182 len 8192 data: 7 9 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 2 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Read 10 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Write 1 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Write 3 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 28 32 Done 28 28 2 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:31.925584212Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:31.925887871Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1552,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 187 187 187 F F F 1 020-039 100 100 100 183 183 183 F F F 2 040-059 96 96 96 175 175 175 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 104 104 87 190 190 159 F F F <--- 5 100-119 104 104 96 190 190 175 F F F <--- 6 120-139 104 104 102 190 190 188 F F T <--- 7 140-159 103 103 103 189 189 189 F F F 8 160-179 103 103 103 189 189 189 T T F <--- 9 180-199 100 100 100 182 182 182 T T F <--- 10 200-219 104 104 102 190 190 187 F F F <--- 11 220-239 104 104 97 190 190 176 F F F <--- 12 240-259 102 102 102 187 187 187 T T F <--- 13 260-279 95 95 95 172 172 172 T T F <--- 14 280-299 100 100 100 182 182 182 F F F 15 300-319 99 99 99 181 181 181 F F F 16 320-339 98 98 98 179 179 179 F F F 17 340-359 102 102 102 187 187 187 F F F 18 360-379 98 98 98 177 177 177 F F F 19 380-399 104 104 102 190 190 186 T T F <--- 20 400-419 96 96 96 174 174 174 F F F 21 420-439 102 102 102 187 187 187 T T F <--- 22 440-459 100 100 100 182 182 182 F F F 23 460-479 102 102 102 188 188 188 F F F 24 480-499 103 103 103 189 189 189 F F F 25 500-519 102 102 102 187 187 187 F F F 26 520-539 104 104 103 191 191 189 F F F <--- 27 540-559 100 100 100 184 184 184 T T F <--- 28 560-579 99 99 99 181 181 181 T T F <--- 29 580-599 102 102 102 187 187 187 F F F Max gen: 104, Max flush: 191 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 187 187 187 F F F 1 020-039 100 100 100 183 183 183 F F F 2 040-059 96 96 96 175 175 175 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 104 104 104 190 190 190 F F F 5 100-119 104 104 104 190 190 190 F F F 6 120-139 104 104 104 190 190 190 F F F 7 140-159 103 103 103 189 189 189 F F F 8 160-179 105 105 105 192 192 192 F F F 9 180-199 105 105 105 192 192 192 F F F 10 200-219 104 104 104 190 190 190 F F F 11 220-239 104 104 104 190 190 190 F F F 12 240-259 105 105 105 192 192 192 F F F 13 260-279 105 105 105 192 192 192 F F F 14 280-299 100 100 100 182 182 182 F F F 15 300-319 99 99 99 181 181 181 F F F 16 320-339 98 98 98 179 179 179 F F F 17 340-359 102 102 102 187 187 187 F F F 18 360-379 98 98 98 177 177 177 F F F 19 380-399 105 105 105 192 192 192 F F F 20 400-419 96 96 96 174 174 174 F F F 21 420-439 105 105 105 192 192 192 F F F 22 440-459 100 100 100 182 182 182 F F F 23 460-479 102 102 102 188 188 188 F F F 24 480-499 103 103 103 189 189 189 F F F 25 500-519 102 102 102 187 187 187 F F F 26 520-539 104 104 104 191 191 191 F F F 27 540-559 105 105 105 192 192 192 F F F 28 560-579 105 105 105 192 192 192 F F F 29 580-599 102 102 102 187 187 187 F F F Max gen: 105, Max flush: 192 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.177071535Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.185134956Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.185420487Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.185457574Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.185466604Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.185478433Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560} {"msg":"Crucible 18e5f65a-6e23-4b8a-857c-ac58a583bafd has session id: 36e6f544-40db-43ad-9056-49eb8bec7f58","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.185784462Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"Upstairs opts: Upstairs UUID: 18e5f65a-6e23-4b8a-857c-ac58a583bafd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.185836988Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"Crucible stats registered with UUID: 18e5f65a-6e23-4b8a-857c-ac58a583bafd","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.185849298Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:36.185859057Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186036805Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186070183Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186081572Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186227702Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.18625254Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560} {"msg":"18e5f65a-6e23-4b8a-857c-ac58a583bafd active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186265199Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186274419Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"client":"0","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186284378Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"client":"1","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186294357Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"client":"2","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186304277Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186314666Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186336634Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"ds_connection connected from Ok(127.0.0.1:53323)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186359873Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"ds_connection connected from Ok(127.0.0.1:63663)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186370772Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"ds_connection connected from Ok(127.0.0.1:65003)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.186386211Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187059735Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"client":"0","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187077884Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"client":"1","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187107372Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"client":"2","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187395952Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187407272Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"[0]R flush_numbers[0..12]: [187, 183, 175, 170, 190, 190, 190, 189, 192, 192, 190, 190]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187447709Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"downstairs","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"[0]R generation[0..12]: [102, 100, 96, 93, 104, 104, 104, 103, 105, 105, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187459288Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"downstairs","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187470577Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"downstairs","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"[1]R flush_numbers[0..12]: [187, 183, 175, 170, 190, 190, 190, 189, 192, 192, 190, 190]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187481506Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"downstairs","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"[1]R generation[0..12]: [102, 100, 96, 93, 104, 104, 104, 103, 105, 105, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187492236Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"downstairs","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187503295Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"downstairs","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"[2]R flush_numbers[0..12]: [187, 183, 175, 170, 190, 190, 190, 189, 192, 192, 190, 190]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187514294Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"downstairs","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"[2]R generation[0..12]: [102, 100, 96, 93, 104, 104, 104, 103, 105, 105, 104, 104]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187525064Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"downstairs","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187536293Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"downstairs","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187546182Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"downstairs","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187556221Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"downstairs","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187567281Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"downstairs","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.18757696Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"":"downstairs","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187587329Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187596429Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"client":"0","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187606458Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"client":"1","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187616357Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"client":"2","":"downstairs","session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187626467Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187635896Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"18e5f65a-6e23-4b8a-857c-ac58a583bafd is now active with session: 36e6f544-40db-43ad-9056-49eb8bec7f58","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187646145Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187655625Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187663684Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"session_id":"36e6f544-40db-43ad-9056-49eb8bec7f58"} {"msg":"Activated sub_volume 18e5f65a-6e23-4b8a-857c-ac58a583bafd","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187673093Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187681563Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.187979423Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"task":"crutest"} 01/30 Write block 349 len 16384 data: 8 7 6 7 02/30 Flush 03/30 Write block 459 len 40960 data: 10 9 8 7 10 11 15 16 12 12 04/30 Write block 218 len 16384 data: 9 9 9 9 05/30 Flush 06/30 Flush 07/30 Write block 49 len 32768 data: 4 5 6 5 3 5 7 8 08/30 Write block 18 len 36864 data: 7 8 8 7 8 6 6 6 6 09/30 Read block 143 len 28672 10/30 Read block 448 len 32768 11/30 Read block 322 len 8192 12/30 Read block 409 len 12288 13/30 Write block 124 len 12288 data: 6 10 10 14/30 Write block 5 len 24576 data: 7 8 10 9 9 8 15/30 Flush 16/30 Write block 23 len 28672 data: 7 7 7 7 6 7 7 17/30 Write block 360 len 40960 data: 11 14 10 9 9 11 12 9 8 8 18/30 Write block 294 len 20480 data: 6 6 6 8 11 19/30 Read block 390 len 32768 20/30 Read block 286 len 40960 21/30 Write block 268 len 20480 data: 6 6 3 3 4 22/30 Read block 542 len 4096 23/30 Flush 24/30 Write block 430 len 4096 data: 9 25/30 Flush 26/30 Write block 154 len 12288 data: 7 6 7 27/30 Write block 428 len 24576 data: 11 9 10 12 12 12 28/30 Read block 406 len 28672 29/30 Read block 68 len 16384 30/30 Write block 532 len 16384 data: 11 11 11 14 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 10 Done Done Done false 1003 Acked Write 4 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Write 9 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 27 29 Done 27 27 1 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:36.196596866Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:36.196899866Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1560,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 102 196 196 187 F F F <--- 1 020-039 106 106 100 197 197 183 F F F <--- 2 040-059 106 106 96 196 196 175 F F F <--- 3 060-079 93 93 93 170 170 170 F F F 4 080-099 104 104 104 190 190 190 F F F 5 100-119 104 104 104 190 190 190 F F F 6 120-139 106 106 104 196 196 190 F F F <--- 7 140-159 103 103 103 189 189 189 T T F <--- 8 160-179 105 105 105 192 192 192 F F F 9 180-199 105 105 105 192 192 192 F F F 10 200-219 106 106 104 194 194 190 F F F <--- 11 220-239 106 106 104 194 194 190 F F F <--- 12 240-259 105 105 105 192 192 192 F F F 13 260-279 106 106 105 197 197 192 F F F <--- 14 280-299 106 106 100 197 197 182 F F F <--- 15 300-319 99 99 99 181 181 181 F F F 16 320-339 98 98 98 179 179 179 F F F 17 340-359 106 106 106 193 193 193 F F F 18 360-379 106 106 98 197 197 177 F F F <--- 19 380-399 105 105 105 192 192 192 F F F 20 400-419 96 96 96 174 174 174 F F F 21 420-439 106 106 105 198 198 192 T T F <--- 22 440-459 106 106 100 194 194 182 F F T <--- 23 460-479 106 106 102 194 194 188 F F T <--- 24 480-499 103 103 103 189 189 189 F F F 25 500-519 102 102 102 187 187 187 F F F 26 520-539 104 104 104 191 191 191 T T F <--- 27 540-559 105 105 105 192 192 192 F F F 28 560-579 105 105 105 192 192 192 F F F 29 580-599 102 102 102 187 187 187 F F F Max gen: 106, Max flush: 198 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 196 196 196 F F F 1 020-039 106 106 106 197 197 197 F F F 2 040-059 106 106 106 196 196 196 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 104 104 104 190 190 190 F F F 5 100-119 104 104 104 190 190 190 F F F 6 120-139 106 106 106 196 196 196 F F F 7 140-159 107 107 107 199 199 199 F F F 8 160-179 105 105 105 192 192 192 F F F 9 180-199 105 105 105 192 192 192 F F F 10 200-219 106 106 106 194 194 194 F F F 11 220-239 106 106 106 194 194 194 F F F 12 240-259 105 105 105 192 192 192 F F F 13 260-279 106 106 106 197 197 197 F F F 14 280-299 106 106 106 197 197 197 F F F 15 300-319 99 99 99 181 181 181 F F F 16 320-339 98 98 98 179 179 179 F F F 17 340-359 106 106 106 193 193 193 F F F 18 360-379 106 106 106 197 197 197 F F F 19 380-399 105 105 105 192 192 192 F F F 20 400-419 96 96 96 174 174 174 F F F 21 420-439 107 107 107 199 199 199 F F F 22 440-459 106 106 106 194 194 194 F F F 23 460-479 106 106 106 194 194 194 F F F 24 480-499 103 103 103 189 189 189 F F F 25 500-519 102 102 102 187 187 187 F F F 26 520-539 107 107 107 199 199 199 F F F 27 540-559 105 105 105 192 192 192 F F F 28 560-579 105 105 105 192 192 192 F F F 29 580-599 102 102 102 187 187 187 F F F Max gen: 107, Max flush: 199 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.444603816Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.452850095Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453123817Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453154354Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453165834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453177783Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568} {"msg":"Crucible d3ec4f81-1935-4559-a8ef-52c98c5b157c has session id: 0ac63de3-6c14-4dd4-b57b-c1b9b164abc4","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453523479Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"Upstairs opts: Upstairs UUID: d3ec4f81-1935-4559-a8ef-52c98c5b157c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453554827Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"Crucible stats registered with UUID: d3ec4f81-1935-4559-a8ef-52c98c5b157c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453566106Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:40.453575826Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453732565Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453751924Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453762463Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453893074Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453912053Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568} {"msg":"d3ec4f81-1935-4559-a8ef-52c98c5b157c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453936561Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453946501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"client":"0","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.45395666Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"client":"1","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453968919Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"client":"2","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453979218Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.453989848Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.454013096Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:57443)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.454070592Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:53550)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.454209973Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"ds_connection connected from Ok(127.0.0.1:49414)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.454221982Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.454617375Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"client":"2","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.454638633Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"client":"0","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.454722008Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"client":"1","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.454897956Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.454922914Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"[0]R flush_numbers[0..12]: [196, 197, 196, 170, 190, 190, 196, 199, 192, 192, 194, 194]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.454970771Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"downstairs","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"[0]R generation[0..12]: [106, 106, 106, 93, 104, 104, 106, 107, 105, 105, 106, 106]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.45498265Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"downstairs","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.454995699Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"downstairs","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"[1]R flush_numbers[0..12]: [196, 197, 196, 170, 190, 190, 196, 199, 192, 192, 194, 194]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455014618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"downstairs","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"[1]R generation[0..12]: [106, 106, 106, 93, 104, 104, 106, 107, 105, 105, 106, 106]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455025567Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"downstairs","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455036796Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"downstairs","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"[2]R flush_numbers[0..12]: [196, 197, 196, 170, 190, 190, 196, 199, 192, 192, 194, 194]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455047906Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"downstairs","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"[2]R generation[0..12]: [106, 106, 106, 93, 104, 104, 106, 107, 105, 105, 106, 106]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455058825Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"downstairs","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455070194Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"downstairs","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455079903Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"downstairs","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455089983Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"downstairs","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455099762Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"downstairs","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455109561Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"":"downstairs","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455119971Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.45512924Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"client":"0","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455139389Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"client":"1","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455149399Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"client":"2","":"downstairs","session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455160888Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455170447Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"d3ec4f81-1935-4559-a8ef-52c98c5b157c is now active with session: 0ac63de3-6c14-4dd4-b57b-c1b9b164abc4","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455180617Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455190066Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455198355Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"session_id":"0ac63de3-6c14-4dd4-b57b-c1b9b164abc4"} {"msg":"Activated sub_volume d3ec4f81-1935-4559-a8ef-52c98c5b157c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455207935Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455216344Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.455543612Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"task":"crutest"} 01/30 Write block 467 len 36864 data: 13 13 12 13 11 9 8 5 5 02/30 Write block 243 len 8192 data: 12 11 03/30 Read block 376 len 20480 04/30 Write block 299 len 8192 data: 10 10 05/30 Read block 202 len 4096 06/30 Write block 467 len 12288 data: 14 14 13 07/30 Read block 384 len 40960 08/30 Read block 129 len 4096 09/30 Read block 523 len 32768 10/30 Read block 511 len 20480 11/30 Read block 264 len 4096 12/30 Read block 304 len 8192 13/30 Write block 349 len 16384 data: 9 8 7 8 14/30 Read block 165 len 16384 15/30 Read block 25 len 12288 16/30 Read block 89 len 16384 17/30 Write block 487 len 12288 data: 12 10 10 18/30 Write block 311 len 16384 data: 14 16 14 14 19/30 Read block 79 len 40960 20/30 Read block 141 len 12288 21/30 Read block 104 len 24576 22/30 Read block 208 len 24576 23/30 Write block 138 len 36864 data: 11 12 10 11 12 13 15 14 14 24/30 Write block 151 len 20480 data: 14 10 7 8 7 25/30 Write block 278 len 16384 data: 4 4 4 5 26/30 Read block 104 len 28672 27/30 Write block 590 len 8192 data: 6 9 28/30 Write block 384 len 24576 data: 8 10 8 5 4 6 29/30 Read block 588 len 24576 30/30 Read block 104 len 32768 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Write 2 Done Sent Done false 1004 Acked Read 1 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 1 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 8 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 0 27 Done 30 3 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:40.463524989Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:40.463809849Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1568,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 196 196 196 F F F 1 020-039 106 106 106 197 197 197 F F F 2 040-059 106 106 106 196 196 196 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 104 104 104 190 190 190 F F F 5 100-119 104 104 104 190 190 190 F F F 6 120-139 106 106 106 196 196 196 T F T <--- 7 140-159 107 107 107 199 199 199 T F T <--- 8 160-179 105 105 105 192 192 192 F F F 9 180-199 105 105 105 192 192 192 F F F 10 200-219 106 106 106 194 194 194 F F F 11 220-239 106 106 106 194 194 194 F F F 12 240-259 105 105 105 192 192 192 T T T 13 260-279 106 106 106 197 197 197 T F T <--- 14 280-299 106 106 106 197 197 197 T F T <--- 15 300-319 99 99 99 181 181 181 T F T <--- 16 320-339 98 98 98 179 179 179 F F F 17 340-359 106 106 106 193 193 193 T F T <--- 18 360-379 106 106 106 197 197 197 F F F 19 380-399 105 105 105 192 192 192 T F T <--- 20 400-419 96 96 96 174 174 174 F F F 21 420-439 107 107 107 199 199 199 F F F 22 440-459 106 106 106 194 194 194 F F F 23 460-479 106 106 106 194 194 194 T T T 24 480-499 103 103 103 189 189 189 T F T <--- 25 500-519 102 102 102 187 187 187 F F F 26 520-539 107 107 107 199 199 199 F F F 27 540-559 105 105 105 192 192 192 F F F 28 560-579 105 105 105 192 192 192 F F F 29 580-599 102 102 102 187 187 187 T F T <--- Max gen: 107, Max flush: 199 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 196 196 196 F F F 1 020-039 106 106 106 197 197 197 F F F 2 040-059 106 106 106 196 196 196 F F F 3 060-079 93 93 93 170 170 170 F F F 4 080-099 104 104 104 190 190 190 F F F 5 100-119 104 104 104 190 190 190 F F F 6 120-139 108 108 108 200 200 200 F F F 7 140-159 108 108 108 200 200 200 F F F 8 160-179 105 105 105 192 192 192 F F F 9 180-199 105 105 105 192 192 192 F F F 10 200-219 106 106 106 194 194 194 F F F 11 220-239 106 106 106 194 194 194 F F F 12 240-259 108 108 108 200 200 200 F F F 13 260-279 108 108 108 200 200 200 F F F 14 280-299 108 108 108 200 200 200 F F F 15 300-319 108 108 108 200 200 200 F F F 16 320-339 98 98 98 179 179 179 F F F 17 340-359 108 108 108 200 200 200 F F F 18 360-379 106 106 106 197 197 197 F F F 19 380-399 108 108 108 200 200 200 F F F 20 400-419 96 96 96 174 174 174 F F F 21 420-439 107 107 107 199 199 199 F F F 22 440-459 106 106 106 194 194 194 F F F 23 460-479 108 108 108 200 200 200 F F F 24 480-499 108 108 108 200 200 200 F F F 25 500-519 102 102 102 187 187 187 F F F 26 520-539 107 107 107 199 199 199 F F F 27 540-559 105 105 105 192 192 192 F F F 28 560-579 105 105 105 192 192 192 F F F 29 580-599 108 108 108 200 200 200 F F F Max gen: 108, Max flush: 200 Begin loop 55 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.706066891Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.714117343Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.714395154Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.714421812Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.714430571Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.714442441Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576} {"msg":"Crucible 362218d8-1984-4729-8d66-daf9ac92e0d9 has session id: bb82983f-b8f1-4fa6-b15d-2159a12b7fc5","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.714765529Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"Upstairs opts: Upstairs UUID: 362218d8-1984-4729-8d66-daf9ac92e0d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.714788197Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"Crucible stats registered with UUID: 362218d8-1984-4729-8d66-daf9ac92e0d9","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.714801096Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:44.714838764Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715018501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.71503625Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715074618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715152942Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715169941Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576} {"msg":"362218d8-1984-4729-8d66-daf9ac92e0d9 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715203389Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715213318Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"client":"0","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715223558Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"client":"1","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715233517Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"client":"2","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715243526Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715253865Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715282683Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57690)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715302362Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"ds_connection connected from Ok(127.0.0.1:38465)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715315501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57477)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715326341Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715935809Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"client":"0","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.715965827Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"client":"1","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716106437Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"client":"2","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716237948Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716262227Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"[0]R flush_numbers[0..12]: [196, 197, 196, 170, 190, 190, 200, 200, 192, 192, 194, 194]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716389428Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"downstairs","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"[0]R generation[0..12]: [106, 106, 106, 93, 104, 104, 108, 108, 105, 105, 106, 106]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716401627Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"downstairs","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716413057Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"downstairs","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"[1]R flush_numbers[0..12]: [196, 197, 196, 170, 190, 190, 200, 200, 192, 192, 194, 194]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716424076Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"downstairs","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"[1]R generation[0..12]: [106, 106, 106, 93, 104, 104, 108, 108, 105, 105, 106, 106]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716434855Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"downstairs","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716446184Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"downstairs","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"[2]R flush_numbers[0..12]: [196, 197, 196, 170, 190, 190, 200, 200, 192, 192, 194, 194]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716456934Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"downstairs","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"[2]R generation[0..12]: [106, 106, 106, 93, 104, 104, 108, 108, 105, 105, 106, 106]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716470093Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"downstairs","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716481292Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"downstairs","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"Max found gen is 109","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716490941Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"downstairs","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"Generation requested: 110 >= found:109","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716501021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"downstairs","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.71651081Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"downstairs","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716520919Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"":"downstairs","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716531259Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716540458Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"client":"0","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716550797Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"client":"1","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716560957Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"client":"2","":"downstairs","session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716571326Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716580885Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"362218d8-1984-4729-8d66-daf9ac92e0d9 is now active with session: bb82983f-b8f1-4fa6-b15d-2159a12b7fc5","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716591174Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716600184Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"session_id":"bb82983f-b8f1-4fa6-b15d-2159a12b7fc5"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716609503Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576} {"msg":"Activated sub_volume 362218d8-1984-4729-8d66-daf9ac92e0d9","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716617713Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716627092Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.716935131Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"task":"crutest"} 01/30 Write block 380 len 12288 data: 11 11 12 02/30 Write block 490 len 32768 data: 8 7 8 10 8 7 7 7 03/30 Read block 508 len 16384 04/30 Write block 558 len 16384 data: 8 10 9 9 05/30 Write block 144 len 40960 data: 16 15 15 14 13 14 14 15 11 8 06/30 Read block 114 len 20480 07/30 Write block 397 len 36864 data: 14 14 14 12 12 12 10 9 9 08/30 Write block 21 len 32768 data: 8 9 8 8 8 8 7 8 09/30 Write block 117 len 20480 data: 5 6 6 5 4 10/30 Read block 329 len 24576 11/30 Write block 355 len 4096 data: 10 12/30 Write block 590 len 40960 data: 7 10 7 7 7 6 6 7 6 2 13/30 Read block 429 len 16384 14/30 Flush 15/30 Write block 108 len 16384 data: 11 10 10 10 16/30 Read block 555 len 8192 17/30 Write block 96 len 40960 data: 9 10 12 11 7 7 6 7 8 6 18/30 Write block 93 len 8192 data: 12 11 19/30 Write block 458 len 16384 data: 13 11 10 9 20/30 Write block 57 len 20480 data: 9 9 8 7 8 21/30 Read block 41 len 20480 22/30 Read block 513 len 4096 23/30 Write block 192 len 40960 data: 9 9 10 11 8 8 9 11 11 10 24/30 Write block 351 len 4096 data: 8 25/30 Read block 529 len 24576 26/30 Flush 27/30 Write block 498 len 20480 data: 6 7 7 7 7 28/30 Read block 153 len 24576 29/30 Write block 256 len 16384 data: 10 11 12 11 30/30 Read block 328 len 20480 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 4 Done Done Sent false 1003 Acked Write 4 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Read 5 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Write 5 Done Done Sent false 1009 Acked Read 6 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Read 5 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 28 28 Done 30 30 2 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:44.725737322Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:44.726026692Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1576,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 196 196 196 F F F 1 020-039 110 110 106 201 201 197 F F F <--- 2 040-059 110 110 106 202 202 196 F F F <--- 3 060-079 110 110 93 202 202 170 F F F <--- 4 080-099 110 110 104 202 202 190 F F F <--- 5 100-119 110 110 104 202 202 190 F F F <--- 6 120-139 110 110 108 201 201 200 F F F <--- 7 140-159 110 110 108 201 201 200 F F F <--- 8 160-179 105 105 105 192 192 192 F F F 9 180-199 110 110 105 202 202 192 F F F <--- 10 200-219 110 110 106 202 202 194 F F F <--- 11 220-239 106 106 106 194 194 194 F F F 12 240-259 108 108 108 200 200 200 T T F <--- 13 260-279 108 108 108 200 200 200 F F F 14 280-299 108 108 108 200 200 200 F F F 15 300-319 108 108 108 200 200 200 F F F 16 320-339 98 98 98 179 179 179 F F F 17 340-359 110 110 108 202 202 200 F F F <--- 18 360-379 106 106 106 197 197 197 F F F 19 380-399 110 110 108 201 201 200 F F T <--- 20 400-419 110 110 96 201 201 174 F F F <--- 21 420-439 107 107 107 199 199 199 F F F 22 440-459 110 110 106 202 202 194 F F F <--- 23 460-479 110 110 108 202 202 200 F F F <--- 24 480-499 110 110 108 201 201 200 T T T <--- 25 500-519 102 102 102 187 187 187 T T F <--- 26 520-539 107 107 107 199 199 199 F F F 27 540-559 110 110 105 201 201 192 F F F <--- 28 560-579 110 110 105 201 201 192 F F F <--- 29 580-599 110 110 108 201 201 200 F F F <--- Max gen: 110, Max flush: 202 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 196 196 196 F F F 1 020-039 110 110 110 201 201 201 F F F 2 040-059 110 110 110 202 202 202 F F F 3 060-079 110 110 110 202 202 202 F F F 4 080-099 110 110 110 202 202 202 F F F 5 100-119 110 110 110 202 202 202 F F F 6 120-139 110 110 110 201 201 201 F F F 7 140-159 110 110 110 201 201 201 F F F 8 160-179 105 105 105 192 192 192 F F F 9 180-199 110 110 110 202 202 202 F F F 10 200-219 110 110 110 202 202 202 F F F 11 220-239 106 106 106 194 194 194 F F F 12 240-259 111 111 111 203 203 203 F F F 13 260-279 108 108 108 200 200 200 F F F 14 280-299 108 108 108 200 200 200 F F F 15 300-319 108 108 108 200 200 200 F F F 16 320-339 98 98 98 179 179 179 F F F 17 340-359 110 110 110 202 202 202 F F F 18 360-379 106 106 106 197 197 197 F F F 19 380-399 110 110 110 201 201 201 F F F 20 400-419 110 110 110 201 201 201 F F F 21 420-439 107 107 107 199 199 199 F F F 22 440-459 110 110 110 202 202 202 F F F 23 460-479 110 110 110 202 202 202 F F F 24 480-499 111 111 111 203 203 203 F F F 25 500-519 111 111 111 203 203 203 F F F 26 520-539 107 107 107 199 199 199 F F F 27 540-559 110 110 110 201 201 201 F F F 28 560-579 110 110 110 201 201 201 F F F 29 580-599 110 110 110 201 201 201 F F F Max gen: 111, Max flush: 203 Begin loop 56 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.981042915Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.989137795Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.989404456Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.989436844Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.989445884Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.989457743Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584} {"msg":"Crucible 7cdf9a58-c1df-4076-beba-a5817346735b has session id: 1c15c15b-558d-4a7f-88bf-51d7af6243f7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.989756102Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"Upstairs opts: Upstairs UUID: 7cdf9a58-c1df-4076-beba-a5817346735b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.98978736Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"Crucible stats registered with UUID: 7cdf9a58-c1df-4076-beba-a5817346735b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.98979909Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:48.989808699Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.989956449Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.989990467Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.990001956Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.990142616Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.990161115Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584} {"msg":"7cdf9a58-c1df-4076-beba-a5817346735b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.990174324Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.990183543Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"client":"0","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.990193713Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"client":"1","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.990203882Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"client":"2","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.990213821Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.990224221Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.99023467Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:44953)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.990333253Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:55811)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.99037513Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:58595)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.99038733Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991007607Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"client":"0","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991030566Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"client":"1","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.99111709Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"client":"2","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991328376Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991352204Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"[0]R flush_numbers[0..12]: [196, 201, 202, 202, 202, 202, 201, 201, 192, 202, 202, 194]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991437288Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"downstairs","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"[0]R generation[0..12]: [106, 110, 110, 110, 110, 110, 110, 110, 105, 110, 110, 106]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991449377Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"downstairs","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991460737Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"downstairs","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"[1]R flush_numbers[0..12]: [196, 201, 202, 202, 202, 202, 201, 201, 192, 202, 202, 194]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991471676Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"downstairs","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"[1]R generation[0..12]: [106, 110, 110, 110, 110, 110, 110, 110, 105, 110, 110, 106]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991482635Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"downstairs","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991493864Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"downstairs","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"[2]R flush_numbers[0..12]: [196, 201, 202, 202, 202, 202, 201, 201, 192, 202, 202, 194]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991504684Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"downstairs","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"[2]R generation[0..12]: [106, 110, 110, 110, 110, 110, 110, 110, 105, 110, 110, 106]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991515393Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"downstairs","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991526592Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"downstairs","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991536441Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"downstairs","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991546561Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"downstairs","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.99155881Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"downstairs","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991568829Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"":"downstairs","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991579268Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991588458Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"client":"0","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991598687Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"client":"1","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991608876Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"client":"2","":"downstairs","session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991619196Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991628735Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"7cdf9a58-c1df-4076-beba-a5817346735b is now active with session: 1c15c15b-558d-4a7f-88bf-51d7af6243f7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991638954Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991648344Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991656353Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"session_id":"1c15c15b-558d-4a7f-88bf-51d7af6243f7"} {"msg":"Activated sub_volume 7cdf9a58-c1df-4076-beba-a5817346735b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991665753Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991674232Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:48.991957863Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"task":"crutest"} 01/30 Write block 359 len 12288 data: 12 12 15 02/30 Read block 383 len 40960 03/30 Read block 453 len 16384 04/30 Write block 313 len 4096 data: 15 05/30 Write block 28 len 16384 data: 9 8 10 12 06/30 Read block 441 len 16384 07/30 Write block 350 len 8192 data: 9 9 08/30 Read block 520 len 4096 09/30 Read block 129 len 8192 10/30 Write block 90 len 20480 data: 7 8 10 13 12 11/30 Write block 69 len 4096 data: 5 12/30 Flush 13/30 Write block 42 len 40960 data: 9 8 8 8 9 8 7 5 6 7 14/30 Write block 559 len 28672 data: 11 10 10 9 10 10 11 15/30 Write block 451 len 12288 data: 6 6 6 16/30 Read block 433 len 36864 17/30 Write block 49 len 32768 data: 6 7 8 6 4 6 8 9 18/30 Write block 239 len 16384 data: 8 9 11 13 19/30 Write block 296 len 40960 data: 7 9 12 11 11 9 9 9 11 10 20/30 Write block 156 len 8192 data: 8 6 21/30 Write block 318 len 36864 data: 17 16 14 13 12 12 11 8 7 22/30 Read block 391 len 16384 23/30 Flush 24/30 Write block 302 len 32768 data: 10 10 12 11 14 14 10 12 25/30 Read block 451 len 36864 26/30 Read block 240 len 4096 27/30 Write block 333 len 24576 data: 14 12 10 10 8 7 28/30 Read block 545 len 24576 29/30 Write block 583 len 28672 data: 8 7 9 8 9 5 5 30/30 Write block 587 len 36864 data: 10 6 6 8 11 8 8 8 7 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Write 1 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Read 1 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 23 27 Done 28 28 7 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:48.999847686Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:49.000342802Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1584,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 196 196 196 F F F 1 020-039 112 112 110 204 204 201 F F T <--- 2 040-059 112 112 110 205 205 202 F F F <--- 3 060-079 112 112 110 204 204 202 F F F <--- 4 080-099 112 112 110 204 204 202 F F F <--- 5 100-119 110 110 110 202 202 202 F F F 6 120-139 110 110 110 201 201 201 F F F 7 140-159 112 112 110 205 205 201 F F F <--- 8 160-179 105 105 105 192 192 192 F F F 9 180-199 110 110 110 202 202 202 F F F 10 200-219 110 110 110 202 202 202 F F F 11 220-239 112 112 106 205 205 194 F F F <--- 12 240-259 112 112 111 205 205 203 F F F <--- 13 260-279 108 108 108 200 200 200 F F F 14 280-299 112 112 108 205 205 200 F F F <--- 15 300-319 112 112 108 205 205 200 T T T <--- 16 320-339 112 112 98 205 205 179 T T F <--- 17 340-359 112 112 110 204 204 202 F F T <--- 18 360-379 112 112 106 204 204 197 F F T <--- 19 380-399 110 110 110 201 201 201 F F F 20 400-419 110 110 110 201 201 201 F F F 21 420-439 107 107 107 199 199 199 F F F 22 440-459 112 112 110 205 205 202 F F F <--- 23 460-479 110 110 110 202 202 202 F F F 24 480-499 111 111 111 203 203 203 F F F 25 500-519 111 111 111 203 203 203 F F F 26 520-539 107 107 107 199 199 199 F F F 27 540-559 112 112 110 205 205 201 F F F <--- 28 560-579 112 112 110 205 205 201 F F F <--- 29 580-599 110 110 110 201 201 201 T T F <--- Max gen: 112, Max flush: 205 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 196 196 196 F F F 1 020-039 112 112 112 204 204 204 F F F 2 040-059 112 112 112 205 205 205 F F F 3 060-079 112 112 112 204 204 204 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 110 110 110 202 202 202 F F F 6 120-139 110 110 110 201 201 201 F F F 7 140-159 112 112 112 205 205 205 F F F 8 160-179 105 105 105 192 192 192 F F F 9 180-199 110 110 110 202 202 202 F F F 10 200-219 110 110 110 202 202 202 F F F 11 220-239 112 112 112 205 205 205 F F F 12 240-259 112 112 112 205 205 205 F F F 13 260-279 108 108 108 200 200 200 F F F 14 280-299 112 112 112 205 205 205 F F F 15 300-319 113 113 113 206 206 206 F F F 16 320-339 113 113 113 206 206 206 F F F 17 340-359 112 112 112 204 204 204 F F F 18 360-379 112 112 112 204 204 204 F F F 19 380-399 110 110 110 201 201 201 F F F 20 400-419 110 110 110 201 201 201 F F F 21 420-439 107 107 107 199 199 199 F F F 22 440-459 112 112 112 205 205 205 F F F 23 460-479 110 110 110 202 202 202 F F F 24 480-499 111 111 111 203 203 203 F F F 25 500-519 111 111 111 203 203 203 F F F 26 520-539 107 107 107 199 199 199 F F F 27 540-559 112 112 112 205 205 205 F F F 28 560-579 112 112 112 205 205 205 F F F 29 580-599 113 113 113 206 206 206 F F F Max gen: 113, Max flush: 206 Begin loop 57 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.249924945Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.25822659Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.258500581Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.25852833Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.258537499Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.258549328Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592} {"msg":"Crucible 5bab020d-45bc-47db-abdb-d2496ae24658 has session id: 13a684f4-44de-458c-8043-d9d9af067eba","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.258901824Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"Upstairs opts: Upstairs UUID: 5bab020d-45bc-47db-abdb-d2496ae24658, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.258923903Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"Crucible stats registered with UUID: 5bab020d-45bc-47db-abdb-d2496ae24658","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.258934062Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:53.258947211Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.25911361Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259144498Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259155777Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259273469Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259292258Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592} {"msg":"5bab020d-45bc-47db-abdb-d2496ae24658 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259316376Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259326045Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"client":"0","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259338314Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"client":"1","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259348444Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"client":"2","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259358423Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259384161Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259414249Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"ds_connection connected from Ok(127.0.0.1:36301)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259442777Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"ds_connection connected from Ok(127.0.0.1:56934)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259454437Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"ds_connection connected from Ok(127.0.0.1:50150)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.259465026Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260128041Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"client":"1","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260145749Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"client":"0","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260228594Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"client":"2","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260456648Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260466818Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"[0]R flush_numbers[0..12]: [196, 204, 205, 204, 204, 202, 201, 205, 192, 202, 202, 205]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260506755Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"downstairs","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"[0]R generation[0..12]: [106, 112, 112, 112, 112, 110, 110, 112, 105, 110, 110, 112]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260518354Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"downstairs","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260531623Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"downstairs","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"[1]R flush_numbers[0..12]: [196, 204, 205, 204, 204, 202, 201, 205, 192, 202, 202, 205]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260542622Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"downstairs","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"[1]R generation[0..12]: [106, 112, 112, 112, 112, 110, 110, 112, 105, 110, 110, 112]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260553502Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"downstairs","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260564751Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"downstairs","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"[2]R flush_numbers[0..12]: [196, 204, 205, 204, 204, 202, 201, 205, 192, 202, 202, 205]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.26057566Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"downstairs","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"[2]R generation[0..12]: [106, 112, 112, 112, 112, 110, 110, 112, 105, 110, 110, 112]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260586659Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"downstairs","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260597839Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"downstairs","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260607758Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"downstairs","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260617847Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"downstairs","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260627407Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"downstairs","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260637076Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"":"downstairs","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260647185Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260661464Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"client":"0","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260671514Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"client":"1","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260682823Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"client":"2","":"downstairs","session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260693042Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260702472Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"5bab020d-45bc-47db-abdb-d2496ae24658 is now active with session: 13a684f4-44de-458c-8043-d9d9af067eba","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260712671Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.26072217Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.26073004Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"session_id":"13a684f4-44de-458c-8043-d9d9af067eba"} {"msg":"Activated sub_volume 5bab020d-45bc-47db-abdb-d2496ae24658","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260739569Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.260747839Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.261077576Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"task":"crutest"} 01/30 Write block 314 len 16384 data: 15 12 15 15 02/30 Read block 351 len 16384 03/30 Flush 04/30 Write block 1 len 36864 data: 4 4 4 6 8 9 11 10 10 05/30 Write block 131 len 32768 data: 7 9 7 9 8 9 10 12 06/30 Read block 57 len 8192 07/30 Write block 451 len 4096 data: 7 08/30 Read block 213 len 8192 09/30 Flush 10/30 Write block 126 len 28672 data: 11 7 8 7 7 8 10 11/30 Read block 511 len 20480 12/30 Write block 270 len 36864 data: 4 4 5 4 3 4 5 4 5 13/30 Read block 390 len 8192 14/30 Read block 430 len 36864 15/30 Read block 139 len 24576 16/30 Read block 106 len 20480 17/30 Write block 73 len 4096 data: 7 18/30 Write block 369 len 28672 data: 9 8 7 10 9 10 12 19/30 Read block 411 len 36864 20/30 Write block 216 len 24576 data: 10 8 10 10 10 10 21/30 Read block 233 len 20480 22/30 Flush 23/30 Write block 323 len 28672 data: 13 12 9 8 8 10 13 24/30 Write block 563 len 8192 data: 11 11 25/30 Write block 1 len 36864 data: 5 5 5 7 9 10 12 11 11 26/30 Read block 37 len 8192 27/30 Write block 368 len 32768 data: 9 10 9 8 11 10 11 13 28/30 Write block 188 len 24576 data: 9 8 8 9 10 10 29/30 Write block 350 len 20480 data: 10 10 9 8 8 30/30 Read block 398 len 12288 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Write 8 Done Done Sent false 1005 Acked Read 2 Done Done Sent false 1006 Acked Write 1 Done Done Sent false 1007 Acked Read 2 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 7 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Write 1 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Read 5 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Write 5 Sent Done Sent false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 28 30 Done 28 30 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:53.269780654Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:53.270070534Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1592,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 106 208 208 196 T T F <--- 1 020-039 112 112 112 204 204 204 F F F 2 040-059 112 112 112 205 205 205 F F F 3 060-079 114 114 112 209 209 204 F F F <--- 4 080-099 112 112 112 204 204 204 F F F 5 100-119 110 110 110 202 202 202 F F F 6 120-139 114 114 110 209 209 201 F F F <--- 7 140-159 112 112 112 205 205 205 F F F 8 160-179 105 105 105 192 192 192 F F F 9 180-199 110 110 110 202 202 202 T T F <--- 10 200-219 114 114 110 209 209 202 F F F <--- 11 220-239 114 114 112 209 209 205 F F F <--- 12 240-259 112 112 112 205 205 205 F F F 13 260-279 114 114 108 209 209 200 F F F <--- 14 280-299 112 112 112 205 205 205 F F F 15 300-319 114 114 113 207 207 206 F F T <--- 16 320-339 113 113 113 206 206 206 T T F <--- 17 340-359 112 112 112 204 204 204 T T F <--- 18 360-379 114 114 112 209 209 204 T T F <--- 19 380-399 110 110 110 201 201 201 F F F 20 400-419 110 110 110 201 201 201 F F F 21 420-439 107 107 107 199 199 199 F F F 22 440-459 114 114 112 208 208 205 F F F <--- 23 460-479 110 110 110 202 202 202 F F F 24 480-499 111 111 111 203 203 203 F F F 25 500-519 111 111 111 203 203 203 F F F 26 520-539 107 107 107 199 199 199 F F F 27 540-559 112 112 112 205 205 205 F F F 28 560-579 112 112 112 205 205 205 T T F <--- 29 580-599 113 113 113 206 206 206 F F F Max gen: 114, Max flush: 209 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 210 210 210 F F F 1 020-039 112 112 112 204 204 204 F F F 2 040-059 112 112 112 205 205 205 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 110 110 110 202 202 202 F F F 6 120-139 114 114 114 209 209 209 F F F 7 140-159 112 112 112 205 205 205 F F F 8 160-179 105 105 105 192 192 192 F F F 9 180-199 115 115 115 210 210 210 F F F 10 200-219 114 114 114 209 209 209 F F F 11 220-239 114 114 114 209 209 209 F F F 12 240-259 112 112 112 205 205 205 F F F 13 260-279 114 114 114 209 209 209 F F F 14 280-299 112 112 112 205 205 205 F F F 15 300-319 114 114 114 207 207 207 F F F 16 320-339 115 115 115 210 210 210 F F F 17 340-359 115 115 115 210 210 210 F F F 18 360-379 115 115 115 210 210 210 F F F 19 380-399 110 110 110 201 201 201 F F F 20 400-419 110 110 110 201 201 201 F F F 21 420-439 107 107 107 199 199 199 F F F 22 440-459 114 114 114 208 208 208 F F F 23 460-479 110 110 110 202 202 202 F F F 24 480-499 111 111 111 203 203 203 F F F 25 500-519 111 111 111 203 203 203 F F F 26 520-539 107 107 107 199 199 199 F F F 27 540-559 112 112 112 205 205 205 F F F 28 560-579 115 115 115 210 210 210 F F F 29 580-599 113 113 113 206 206 206 F F F Max gen: 115, Max flush: 210 Begin loop 58 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.51520551Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.523366554Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.523647215Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.523673243Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.523682363Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.523694182Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600} {"msg":"Crucible 02ceaba4-4e6d-40ff-a675-0abe01152b19 has session id: 9f313869-61a5-489b-9ecc-1c38ee723817","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524030919Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"Upstairs opts: Upstairs UUID: 02ceaba4-4e6d-40ff-a675-0abe01152b19, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524051448Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"Crucible stats registered with UUID: 02ceaba4-4e6d-40ff-a675-0abe01152b19","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524061687Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:57.524074456Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524271723Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524301161Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.5243124Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.52445528Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524479699Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600} {"msg":"02ceaba4-4e6d-40ff-a675-0abe01152b19 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524498287Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524507827Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"client":"0","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524518286Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"client":"1","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524528345Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"client":"2","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524553673Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524564783Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524575102Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"ds_connection connected from Ok(127.0.0.1:52043)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524660176Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"ds_connection connected from Ok(127.0.0.1:37575)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524799947Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"ds_connection connected from Ok(127.0.0.1:44354)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.524859733Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.52534452Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"client":"0","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525362578Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"client":"2","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525374318Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"client":"1","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.52563831Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525660008Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"[0]R flush_numbers[0..12]: [210, 204, 205, 209, 204, 202, 209, 205, 192, 210, 209, 209]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525723024Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"downstairs","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"[0]R generation[0..12]: [115, 112, 112, 114, 112, 110, 114, 112, 105, 115, 114, 114]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525735053Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"downstairs","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525746722Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"downstairs","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"[1]R flush_numbers[0..12]: [210, 204, 205, 209, 204, 202, 209, 205, 192, 210, 209, 209]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525757852Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"downstairs","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"[1]R generation[0..12]: [115, 112, 112, 114, 112, 110, 114, 112, 105, 115, 114, 114]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525768651Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"downstairs","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.52577987Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"downstairs","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"[2]R flush_numbers[0..12]: [210, 204, 205, 209, 204, 202, 209, 205, 192, 210, 209, 209]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525792979Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"downstairs","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"[2]R generation[0..12]: [115, 112, 112, 114, 112, 110, 114, 112, 105, 115, 114, 114]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525803878Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"downstairs","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525815158Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"downstairs","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525824787Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"downstairs","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525834896Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"downstairs","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525844736Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"downstairs","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525854255Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"":"downstairs","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525864524Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525873834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"client":"0","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525884163Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"client":"1","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525894292Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"client":"2","":"downstairs","session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525904642Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525914141Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"02ceaba4-4e6d-40ff-a675-0abe01152b19 is now active with session: 9f313869-61a5-489b-9ecc-1c38ee723817","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.52592437Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.52593339Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"session_id":"9f313869-61a5-489b-9ecc-1c38ee723817"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525942879Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600} {"msg":"Activated sub_volume 02ceaba4-4e6d-40ff-a675-0abe01152b19","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525952328Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.525960758Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.526292945Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"task":"crutest"} 01/30 Write block 287 len 12288 data: 6 4 3 02/30 Read block 3 len 16384 03/30 Write block 107 len 16384 data: 10 12 11 11 04/30 Read block 163 len 28672 05/30 Read block 354 len 12288 06/30 Write block 310 len 8192 data: 12 15 07/30 Read block 415 len 16384 08/30 Write block 221 len 20480 data: 11 8 7 10 8 09/30 Write block 562 len 28672 data: 10 12 12 12 9 11 10 10/30 Read block 87 len 24576 11/30 Write block 233 len 16384 data: 7 6 8 8 12/30 Write block 500 len 24576 data: 8 8 8 8 10 12 13/30 Read block 240 len 24576 14/30 Read block 588 len 36864 15/30 Write block 36 len 40960 data: 10 8 9 11 11 10 10 9 9 9 16/30 Write block 523 len 20480 data: 9 11 13 11 10 17/30 Write block 327 len 16384 data: 9 11 14 11 18/30 Read block 98 len 16384 19/30 Read block 59 len 28672 20/30 Write block 219 len 16384 data: 11 11 12 9 21/30 Read block 480 len 8192 22/30 Flush 23/30 Write block 307 len 8192 data: 15 11 24/30 Write block 420 len 32768 data: 10 9 9 8 7 8 9 10 25/30 Write block 183 len 16384 data: 10 8 9 7 26/30 Flush 27/30 Write block 265 len 8192 data: 7 5 28/30 Write block 510 len 28672 data: 11 9 5 7 7 6 5 29/30 Read block 514 len 36864 30/30 Write block 526 len 36864 data: 12 11 10 10 10 10 12 12 12 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Sent Done false 1001 Acked Read 4 Done Sent Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Read 7 Done Sent Done false 1004 Acked Read 3 Done Sent Done false 1005 Acked Write 2 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Write 4 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 2 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:42:57.534674975Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:42:57.534967215Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1600,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 210 210 210 F F F 1 020-039 116 112 116 211 204 211 F F F <--- 2 040-059 116 112 116 211 205 211 F F F <--- 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 116 110 116 211 202 211 F F F <--- 6 120-139 114 114 114 209 209 209 F F F 7 140-159 112 112 112 205 205 205 F F F 8 160-179 105 105 105 192 192 192 F F F 9 180-199 116 115 116 212 210 212 F F F <--- 10 200-219 116 114 116 211 209 211 F F F <--- 11 220-239 116 114 116 211 209 211 F F F <--- 12 240-259 112 112 112 205 205 205 F F F 13 260-279 114 114 114 209 209 209 T F T <--- 14 280-299 116 112 116 211 205 211 F F F <--- 15 300-319 116 114 116 212 207 212 F F F <--- 16 320-339 116 115 116 211 210 211 F F F <--- 17 340-359 115 115 115 210 210 210 F F F 18 360-379 115 115 115 210 210 210 F F F 19 380-399 110 110 110 201 201 201 F F F 20 400-419 110 110 110 201 201 201 F F F 21 420-439 116 107 116 212 199 212 F F F <--- 22 440-459 114 114 114 208 208 208 F F F 23 460-479 110 110 110 202 202 202 F F F 24 480-499 111 111 111 203 203 203 F F F 25 500-519 116 111 116 211 203 211 T F T <--- 26 520-539 116 107 116 211 199 211 T F T <--- 27 540-559 112 112 112 205 205 205 F F F 28 560-579 116 115 116 211 210 211 F F F <--- 29 580-599 113 113 113 206 206 206 F F F Max gen: 116, Max flush: 212 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 210 210 210 F F F 1 020-039 116 116 116 211 211 211 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 116 116 116 211 211 211 F F F 6 120-139 114 114 114 209 209 209 F F F 7 140-159 112 112 112 205 205 205 F F F 8 160-179 105 105 105 192 192 192 F F F 9 180-199 116 116 116 212 212 212 F F F 10 200-219 116 116 116 211 211 211 F F F 11 220-239 116 116 116 211 211 211 F F F 12 240-259 112 112 112 205 205 205 F F F 13 260-279 117 117 117 213 213 213 F F F 14 280-299 116 116 116 211 211 211 F F F 15 300-319 116 116 116 212 212 212 F F F 16 320-339 116 116 116 211 211 211 F F F 17 340-359 115 115 115 210 210 210 F F F 18 360-379 115 115 115 210 210 210 F F F 19 380-399 110 110 110 201 201 201 F F F 20 400-419 110 110 110 201 201 201 F F F 21 420-439 116 116 116 212 212 212 F F F 22 440-459 114 114 114 208 208 208 F F F 23 460-479 110 110 110 202 202 202 F F F 24 480-499 111 111 111 203 203 203 F F F 25 500-519 117 117 117 213 213 213 F F F 26 520-539 117 117 117 213 213 213 F F F 27 540-559 112 112 112 205 205 205 F F F 28 560-579 116 116 116 211 211 211 F F F 29 580-599 113 113 113 206 206 206 F F F Max gen: 117, Max flush: 213 Begin loop 59 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.781640816Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.78980942Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.79009326Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790120159Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790129358Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790141317Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608} {"msg":"Crucible b2071948-5234-4f2b-9389-77f83ca17b98 has session id: 521b3a94-f09b-4259-9e5a-c56445016573","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790513292Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"Upstairs opts: Upstairs UUID: b2071948-5234-4f2b-9389-77f83ca17b98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.79054237Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"Crucible stats registered with UUID: b2071948-5234-4f2b-9389-77f83ca17b98","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790553739Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:01.790563508Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790725747Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790757795Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790769114Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790883897Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790908785Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608} {"msg":"b2071948-5234-4f2b-9389-77f83ca17b98 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790926474Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790935743Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"client":"0","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790946002Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"client":"1","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790956172Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"client":"2","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.790966161Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.79097661Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.7909872Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"ds_connection connected from Ok(127.0.0.1:53992)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.791080723Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"ds_connection connected from Ok(127.0.0.1:60090)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.791150539Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"ds_connection connected from Ok(127.0.0.1:38769)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.791162848Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.791738729Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"client":"1","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.791770926Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"client":"0","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.791818393Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"client":"2","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792053187Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792082355Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"[0]R flush_numbers[0..12]: [210, 211, 211, 209, 204, 211, 209, 205, 192, 212, 211, 211]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792139701Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"downstairs","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"[0]R generation[0..12]: [115, 116, 116, 114, 112, 116, 114, 112, 105, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.79215126Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"downstairs","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.79216286Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"downstairs","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"[1]R flush_numbers[0..12]: [210, 211, 211, 209, 204, 211, 209, 205, 192, 212, 211, 211]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792173799Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"downstairs","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"[1]R generation[0..12]: [115, 116, 116, 114, 112, 116, 114, 112, 105, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792184788Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"downstairs","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792195947Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"downstairs","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"[2]R flush_numbers[0..12]: [210, 211, 211, 209, 204, 211, 209, 205, 192, 212, 211, 211]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792206717Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"downstairs","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"[2]R generation[0..12]: [115, 116, 116, 114, 112, 116, 114, 112, 105, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792217586Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"downstairs","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792228695Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"downstairs","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792238335Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"downstairs","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792250184Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"downstairs","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792259753Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"downstairs","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792269422Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"":"downstairs","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792279632Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792288711Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"client":"0","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.79229878Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"client":"1","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.79230879Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"client":"2","":"downstairs","session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792318929Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792328358Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"b2071948-5234-4f2b-9389-77f83ca17b98 is now active with session: 521b3a94-f09b-4259-9e5a-c56445016573","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792338508Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792347997Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792355947Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"session_id":"521b3a94-f09b-4259-9e5a-c56445016573"} {"msg":"Activated sub_volume b2071948-5234-4f2b-9389-77f83ca17b98","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792365426Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792373905Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.792645107Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"task":"crutest"} 01/30 Write block 314 len 16384 data: 16 13 16 16 02/30 Read block 221 len 36864 03/30 Read block 218 len 24576 04/30 Read block 10 len 12288 05/30 Read block 149 len 16384 06/30 Read block 422 len 40960 07/30 Read block 271 len 4096 08/30 Read block 291 len 36864 09/30 Read block 549 len 4096 10/30 Read block 494 len 36864 11/30 Read block 351 len 20480 12/30 Read block 575 len 36864 13/30 Write block 116 len 16384 data: 7 6 7 7 14/30 Write block 259 len 20480 data: 12 10 9 8 9 15/30 Write block 161 len 28672 data: 10 10 10 11 12 12 14 16/30 Read block 169 len 16384 17/30 Write block 139 len 24576 data: 13 11 12 13 14 17 18/30 Read block 230 len 4096 19/30 Read block 542 len 24576 20/30 Read block 172 len 8192 21/30 Flush 22/30 Write block 544 len 24576 data: 10 10 10 10 11 11 23/30 Read block 189 len 4096 24/30 Write block 275 len 32768 data: 5 6 5 6 5 5 6 6 25/30 Read block 351 len 12288 26/30 Read block 479 len 24576 27/30 Write block 478 len 8192 data: 9 7 28/30 Read block 485 len 12288 29/30 Read block 140 len 28672 30/30 Read block 500 len 40960 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Sent false 1001 Acked Read 9 Done Done Sent false 1002 Acked Read 6 Done Done Sent false 1003 Acked Read 3 Done Done Sent false 1004 Acked Read 4 Done Done Sent false 1005 Acked Read 10 Done Done Sent false 1006 Acked Read 1 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Write 4 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Read 6 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 10 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:01.80127059Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:01.801556081Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1608,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 210 210 210 F F F 1 020-039 116 116 116 211 211 211 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 118 118 116 214 214 211 F F F <--- 6 120-139 118 118 114 214 214 209 F F F <--- 7 140-159 118 118 112 214 214 205 F F F <--- 8 160-179 118 118 105 214 214 192 F F F <--- 9 180-199 116 116 116 212 212 212 F F F 10 200-219 116 116 116 211 211 211 F F F 11 220-239 116 116 116 211 211 211 F F F 12 240-259 118 118 112 214 214 205 F F F <--- 13 260-279 118 118 117 214 214 213 T T F <--- 14 280-299 116 116 116 211 211 211 T T F <--- 15 300-319 118 118 116 214 214 212 F F F <--- 16 320-339 116 116 116 211 211 211 F F F 17 340-359 115 115 115 210 210 210 F F F 18 360-379 115 115 115 210 210 210 F F F 19 380-399 110 110 110 201 201 201 F F F 20 400-419 110 110 110 201 201 201 F F F 21 420-439 116 116 116 212 212 212 F F F 22 440-459 114 114 114 208 208 208 F F F 23 460-479 110 110 110 202 202 202 T T F <--- 24 480-499 111 111 111 203 203 203 F F F 25 500-519 117 117 117 213 213 213 F F F 26 520-539 117 117 117 213 213 213 F F F 27 540-559 112 112 112 205 205 205 T T F <--- 28 560-579 116 116 116 211 211 211 F F F 29 580-599 113 113 113 206 206 206 F F F Max gen: 118, Max flush: 214 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 210 210 210 F F F 1 020-039 116 116 116 211 211 211 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 118 118 118 214 214 214 F F F 6 120-139 118 118 118 214 214 214 F F F 7 140-159 118 118 118 214 214 214 F F F 8 160-179 118 118 118 214 214 214 F F F 9 180-199 116 116 116 212 212 212 F F F 10 200-219 116 116 116 211 211 211 F F F 11 220-239 116 116 116 211 211 211 F F F 12 240-259 118 118 118 214 214 214 F F F 13 260-279 119 119 119 215 215 215 F F F 14 280-299 119 119 119 215 215 215 F F F 15 300-319 118 118 118 214 214 214 F F F 16 320-339 116 116 116 211 211 211 F F F 17 340-359 115 115 115 210 210 210 F F F 18 360-379 115 115 115 210 210 210 F F F 19 380-399 110 110 110 201 201 201 F F F 20 400-419 110 110 110 201 201 201 F F F 21 420-439 116 116 116 212 212 212 F F F 22 440-459 114 114 114 208 208 208 F F F 23 460-479 119 119 119 215 215 215 F F F 24 480-499 111 111 111 203 203 203 F F F 25 500-519 117 117 117 213 213 213 F F F 26 520-539 117 117 117 213 213 213 F F F 27 540-559 119 119 119 215 215 215 F F F 28 560-579 116 116 116 211 211 211 F F F 29 580-599 113 113 113 206 206 206 F F F Max gen: 119, Max flush: 215 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.038063572Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.046359807Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.046652647Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.046679316Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.046688425Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.046700414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616} {"msg":"Crucible fd1750a5-2c0d-4c0d-ab31-74bb9a4ce12d has session id: 98ac6de9-5b78-4854-a242-1b66448a3c13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.04705127Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"Upstairs opts: Upstairs UUID: fd1750a5-2c0d-4c0d-ab31-74bb9a4ce12d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047081418Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"Crucible stats registered with UUID: fd1750a5-2c0d-4c0d-ab31-74bb9a4ce12d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047092897Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:06.047102827Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047265456Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047282315Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047292934Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047410476Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047425475Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616} {"msg":"fd1750a5-2c0d-4c0d-ab31-74bb9a4ce12d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047442354Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047451603Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"client":"0","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047463452Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"client":"1","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047473631Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"client":"2","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047483641Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.04749411Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047532008Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"ds_connection connected from Ok(127.0.0.1:57686)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047556156Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"ds_connection connected from Ok(127.0.0.1:33855)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047567835Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"ds_connection connected from Ok(127.0.0.1:54456)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.047615962Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048254978Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"client":"1","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048288376Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"client":"0","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048406438Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"client":"2","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048594235Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048604455Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"[0]R flush_numbers[0..12]: [210, 211, 211, 209, 204, 214, 214, 214, 214, 212, 211, 211]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048711037Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"downstairs","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"[0]R generation[0..12]: [115, 116, 116, 114, 112, 118, 118, 118, 118, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048722537Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"downstairs","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048736196Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"downstairs","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"[1]R flush_numbers[0..12]: [210, 211, 211, 209, 204, 214, 214, 214, 214, 212, 211, 211]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048747225Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"downstairs","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"[1]R generation[0..12]: [115, 116, 116, 114, 112, 118, 118, 118, 118, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048758044Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"downstairs","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048769303Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"downstairs","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"[2]R flush_numbers[0..12]: [210, 211, 211, 209, 204, 214, 214, 214, 214, 212, 211, 211]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048780303Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"downstairs","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"[2]R generation[0..12]: [115, 116, 116, 114, 112, 118, 118, 118, 118, 116, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048791112Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"downstairs","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048802371Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"downstairs","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.04881218Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"downstairs","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.04882272Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"downstairs","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048832529Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"downstairs","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048842398Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"":"downstairs","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048852428Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048861517Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"client":"0","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048871466Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"client":"1","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048881246Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"client":"2","":"downstairs","session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048892515Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048902024Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"fd1750a5-2c0d-4c0d-ab31-74bb9a4ce12d is now active with session: 98ac6de9-5b78-4854-a242-1b66448a3c13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048912334Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048921773Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048929762Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"session_id":"98ac6de9-5b78-4854-a242-1b66448a3c13"} {"msg":"Activated sub_volume fd1750a5-2c0d-4c0d-ab31-74bb9a4ce12d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048939282Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.048947661Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.049304777Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"task":"crutest"} 01/30 Write block 229 len 36864 data: 8 8 10 10 8 7 9 9 6 02/30 Write block 513 len 40960 data: 8 8 7 6 4 5 7 9 9 10 03/30 Read block 518 len 28672 04/30 Read block 380 len 28672 05/30 Write block 451 len 24576 data: 8 7 7 8 8 8 06/30 Write block 412 len 4096 data: 8 07/30 Write block 481 len 32768 data: 7 8 9 10 10 10 13 11 08/30 Flush 09/30 Write block 334 len 28672 data: 13 11 11 9 8 8 8 10/30 Read block 137 len 40960 11/30 Read block 440 len 8192 12/30 Read block 356 len 40960 13/30 Read block 233 len 20480 14/30 Read block 407 len 32768 15/30 Read block 474 len 20480 16/30 Read block 241 len 4096 17/30 Flush 18/30 Write block 315 len 40960 data: 14 17 17 18 17 15 14 13 14 13 19/30 Read block 503 len 36864 20/30 Read block 576 len 28672 21/30 Flush 22/30 Write block 499 len 28672 data: 8 9 9 9 9 11 13 23/30 Read block 268 len 12288 24/30 Read block 187 len 32768 25/30 Write block 200 len 28672 data: 12 11 8 8 11 12 12 26/30 Write block 159 len 40960 data: 9 9 11 11 11 12 13 13 15 12 27/30 Write block 535 len 36864 data: 15 12 12 13 13 10 9 11 9 28/30 Write block 225 len 28672 data: 9 10 11 13 9 9 11 29/30 Write block 338 len 36864 data: 9 9 9 9 10 10 11 10 7 30/30 Write block 178 len 8192 data: 5 6 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 10 Done Done Sent false 1002 Acked Read 7 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Write 10 Sent Sent Sent false 1026 Acked Write 9 Sent Sent Sent false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 5 5 29 39 Done 25 25 1 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:06.057657009Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:06.057949509Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1616,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 210 210 210 F F F 1 020-039 116 116 116 211 211 211 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 118 118 118 214 214 214 F F F 6 120-139 118 118 118 214 214 214 F F F 7 140-159 118 118 118 214 214 214 T T F <--- 8 160-179 118 118 118 214 214 214 T T F <--- 9 180-199 116 116 116 212 212 212 F F F 10 200-219 116 116 116 211 211 211 T T F <--- 11 220-239 120 120 116 216 216 211 T T T <--- 12 240-259 118 118 118 214 214 214 F F F 13 260-279 119 119 119 215 215 215 F F F 14 280-299 119 119 119 215 215 215 F F F 15 300-319 120 120 118 218 218 214 F F F <--- 16 320-339 120 120 116 218 218 211 T T F <--- 17 340-359 120 120 115 217 217 210 T T F <--- 18 360-379 115 115 115 210 210 210 F F F 19 380-399 110 110 110 201 201 201 F F F 20 400-419 120 120 110 216 216 201 F F F <--- 21 420-439 116 116 116 212 212 212 F F F 22 440-459 120 120 114 216 216 208 F F F <--- 23 460-479 119 119 119 215 215 215 F F F 24 480-499 120 120 111 216 216 203 T T F <--- 25 500-519 120 120 117 216 216 213 T T F <--- 26 520-539 120 120 117 216 216 213 T T F <--- 27 540-559 119 119 119 215 215 215 T T F <--- 28 560-579 116 116 116 211 211 211 F F F 29 580-599 113 113 113 206 206 206 F F F Max gen: 120, Max flush: 218 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 210 210 210 F F F 1 020-039 116 116 116 211 211 211 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 118 118 118 214 214 214 F F F 6 120-139 118 118 118 214 214 214 F F F 7 140-159 121 121 121 219 219 219 F F F 8 160-179 121 121 121 219 219 219 F F F 9 180-199 116 116 116 212 212 212 F F F 10 200-219 121 121 121 219 219 219 F F F 11 220-239 121 121 121 219 219 219 F F F 12 240-259 118 118 118 214 214 214 F F F 13 260-279 119 119 119 215 215 215 F F F 14 280-299 119 119 119 215 215 215 F F F 15 300-319 120 120 120 218 218 218 F F F 16 320-339 121 121 121 219 219 219 F F F 17 340-359 121 121 121 219 219 219 F F F 18 360-379 115 115 115 210 210 210 F F F 19 380-399 110 110 110 201 201 201 F F F 20 400-419 120 120 120 216 216 216 F F F 21 420-439 116 116 116 212 212 212 F F F 22 440-459 120 120 120 216 216 216 F F F 23 460-479 119 119 119 215 215 215 F F F 24 480-499 121 121 121 219 219 219 F F F 25 500-519 121 121 121 219 219 219 F F F 26 520-539 121 121 121 219 219 219 F F F 27 540-559 121 121 121 219 219 219 F F F 28 560-579 116 116 116 211 211 211 F F F 29 580-599 113 113 113 206 206 206 F F F Max gen: 121, Max flush: 219 Begin loop 61 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.3097523Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.317923334Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.318204945Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.318231923Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.318240833Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.318252702Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624} {"msg":"Crucible 96f33b2c-d846-477b-9637-f9f2c1665b69 has session id: e56edecb-dca0-4f16-ba53-8c3613eccbf0","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.318637426Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"Upstairs opts: Upstairs UUID: 96f33b2c-d846-477b-9637-f9f2c1665b69, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.318657504Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"Crucible stats registered with UUID: 96f33b2c-d846-477b-9637-f9f2c1665b69","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.318667844Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:10.318680243Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.318856691Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.318888569Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.318899758Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319033979Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319051178Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624} {"msg":"96f33b2c-d846-477b-9637-f9f2c1665b69 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319068436Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319077716Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"client":"0","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319087665Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"client":"1","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319097824Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"client":"2","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319107794Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319118213Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319128472Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"ds_connection connected from Ok(127.0.0.1:40969)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319219346Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"ds_connection connected from Ok(127.0.0.1:40811)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319287572Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46602)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319299781Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.31990181Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"client":"1","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319917049Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"client":"2","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.319935977Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"client":"0","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320239917Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320252336Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"[0]R flush_numbers[0..12]: [210, 211, 211, 209, 204, 214, 214, 219, 219, 212, 219, 219]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320290953Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"downstairs","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"[0]R generation[0..12]: [115, 116, 116, 114, 112, 118, 118, 121, 121, 116, 121, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320302533Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"downstairs","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320313882Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"downstairs","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"[1]R flush_numbers[0..12]: [210, 211, 211, 209, 204, 214, 214, 219, 219, 212, 219, 219]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320324751Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"downstairs","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"[1]R generation[0..12]: [115, 116, 116, 114, 112, 118, 118, 121, 121, 116, 121, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.32033567Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"downstairs","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320346979Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"downstairs","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"[2]R flush_numbers[0..12]: [210, 211, 211, 209, 204, 214, 214, 219, 219, 212, 219, 219]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320360289Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"downstairs","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"[2]R generation[0..12]: [115, 116, 116, 114, 112, 118, 118, 121, 121, 116, 121, 121]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320371268Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"downstairs","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320382467Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"downstairs","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320392196Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"downstairs","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320402336Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"downstairs","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320412075Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"downstairs","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320421924Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"":"downstairs","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320432304Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320441633Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"client":"0","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320451942Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"client":"1","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320462002Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"client":"2","":"downstairs","session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320472371Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.32048181Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"96f33b2c-d846-477b-9637-f9f2c1665b69 is now active with session: e56edecb-dca0-4f16-ba53-8c3613eccbf0","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.3204921Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320501589Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"session_id":"e56edecb-dca0-4f16-ba53-8c3613eccbf0"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320511068Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624} {"msg":"Activated sub_volume 96f33b2c-d846-477b-9637-f9f2c1665b69","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320520518Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320529017Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.320856235Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"task":"crutest"} 01/30 Write block 20 len 20480 data: 9 9 10 9 9 02/30 Flush 03/30 Flush 04/30 Write block 210 len 40960 data: 11 12 11 10 8 10 11 9 11 12 05/30 Write block 531 len 40960 data: 11 13 13 13 16 13 13 14 14 11 06/30 Write block 116 len 36864 data: 8 7 8 8 6 5 5 5 7 07/30 Write block 215 len 24576 data: 11 12 10 12 13 12 08/30 Write block 474 len 16384 data: 6 6 6 6 09/30 Write block 475 len 36864 data: 7 7 7 10 8 7 8 9 10 10/30 Read block 271 len 36864 11/30 Read block 302 len 20480 12/30 Read block 309 len 36864 13/30 Write block 146 len 12288 data: 16 15 14 14/30 Write block 250 len 24576 data: 6 6 5 5 8 9 15/30 Read block 222 len 20480 16/30 Flush 17/30 Flush 18/30 Write block 340 len 8192 data: 10 10 19/30 Flush 20/30 Write block 290 len 40960 data: 2 3 5 6 7 7 8 10 13 12 21/30 Write block 408 len 4096 data: 9 22/30 Write block 411 len 24576 data: 9 9 10 10 12 13 23/30 Read block 493 len 20480 24/30 Read block 375 len 36864 25/30 Read block 136 len 36864 26/30 Write block 119 len 16384 data: 9 7 6 6 27/30 Read block 451 len 24576 28/30 Write block 467 len 36864 data: 15 15 14 14 12 10 9 7 8 29/30 Read block 105 len 16384 30/30 Read block 280 len 8192 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Write 4 Done Done Sent false 1008 Acked Write 9 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 2 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 1 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Read 9 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 28 28 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1018)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:10.329079565Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:10.329364896Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1624,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 210 210 210 F F F 1 020-039 122 122 122 220 220 220 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 122 122 118 222 222 214 T T F <--- 6 120-139 122 122 118 222 222 214 T T F <--- 7 140-159 122 122 121 222 222 219 F F F <--- 8 160-179 121 121 121 219 219 219 F F F 9 180-199 116 116 116 212 212 212 F F F 10 200-219 122 122 121 222 222 219 F F F <--- 11 220-239 122 122 121 222 222 219 F F F <--- 12 240-259 122 122 118 222 222 214 F F F <--- 13 260-279 119 119 119 215 215 215 F F F 14 280-299 119 119 119 215 215 215 T T F <--- 15 300-319 120 120 120 218 218 218 F F F 16 320-339 121 121 121 219 219 219 F F F 17 340-359 122 122 121 224 224 219 F F F <--- 18 360-379 115 115 115 210 210 210 F F F 19 380-399 110 110 110 201 201 201 F F F 20 400-419 120 120 120 216 216 216 T T F <--- 21 420-439 116 116 116 212 212 212 F F F 22 440-459 120 120 120 216 216 216 F F F 23 460-479 122 122 119 222 222 215 T T F <--- 24 480-499 122 122 121 222 222 219 F F F <--- 25 500-519 121 121 121 219 219 219 F F F 26 520-539 122 122 121 222 222 219 F F F <--- 27 540-559 122 122 121 222 222 219 F F F <--- 28 560-579 116 116 116 211 211 211 F F F 29 580-599 113 113 113 206 206 206 F F F Max gen: 122, Max flush: 224 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 210 210 210 F F F 1 020-039 122 122 122 220 220 220 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 123 123 123 225 225 225 F F F 6 120-139 123 123 123 225 225 225 F F F 7 140-159 122 122 122 222 222 222 F F F 8 160-179 121 121 121 219 219 219 F F F 9 180-199 116 116 116 212 212 212 F F F 10 200-219 122 122 122 222 222 222 F F F 11 220-239 122 122 122 222 222 222 F F F 12 240-259 122 122 122 222 222 222 F F F 13 260-279 119 119 119 215 215 215 F F F 14 280-299 123 123 123 225 225 225 F F F 15 300-319 120 120 120 218 218 218 F F F 16 320-339 121 121 121 219 219 219 F F F 17 340-359 122 122 122 224 224 224 F F F 18 360-379 115 115 115 210 210 210 F F F 19 380-399 110 110 110 201 201 201 F F F 20 400-419 123 123 123 225 225 225 F F F 21 420-439 116 116 116 212 212 212 F F F 22 440-459 120 120 120 216 216 216 F F F 23 460-479 123 123 123 225 225 225 F F F 24 480-499 122 122 122 222 222 222 F F F 25 500-519 121 121 121 219 219 219 F F F 26 520-539 122 122 122 222 222 222 F F F 27 540-559 122 122 122 222 222 222 F F F 28 560-579 116 116 116 211 211 211 F F F 29 580-599 113 113 113 206 206 206 F F F Max gen: 123, Max flush: 225 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.573741453Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.58186802Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.58215854Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.582185628Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.582194908Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.582206887Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632} {"msg":"Crucible fe30f370-f616-40df-bd12-0ce20117caa5 has session id: 82d1dd08-832a-403f-a52e-3743cec165b9","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.58260798Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"Upstairs opts: Upstairs UUID: fe30f370-f616-40df-bd12-0ce20117caa5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.582628038Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"Crucible stats registered with UUID: fe30f370-f616-40df-bd12-0ce20117caa5","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.582638328Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:14.582648337Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.582823605Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.582858373Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.582869842Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.582947436Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.582965625Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632} {"msg":"fe30f370-f616-40df-bd12-0ce20117caa5 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.582992803Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.583011842Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"client":"0","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.583021931Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"client":"1","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.583031881Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"client":"2","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.58304173Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.583052309Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.583062749Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:34813)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.583156942Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:62208)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.583237617Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:53650)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.583262335Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.583862734Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"client":"0","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.583908271Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"client":"2","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.58392248Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"client":"1","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584185862Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.58422098Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"[0]R flush_numbers[0..12]: [210, 220, 211, 209, 204, 225, 225, 222, 219, 212, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584287625Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"downstairs","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"[0]R generation[0..12]: [115, 122, 116, 114, 112, 123, 123, 122, 121, 116, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584299195Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"downstairs","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584310444Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"downstairs","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"[1]R flush_numbers[0..12]: [210, 220, 211, 209, 204, 225, 225, 222, 219, 212, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584321243Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"downstairs","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"[1]R generation[0..12]: [115, 122, 116, 114, 112, 123, 123, 122, 121, 116, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584332062Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"downstairs","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584343242Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"downstairs","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"[2]R flush_numbers[0..12]: [210, 220, 211, 209, 204, 225, 225, 222, 219, 212, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584354131Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"downstairs","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"[2]R generation[0..12]: [115, 122, 116, 114, 112, 123, 123, 122, 121, 116, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.58436493Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"downstairs","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584375989Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"downstairs","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584385629Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"downstairs","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584395838Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"downstairs","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584407117Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"downstairs","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584416827Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"":"downstairs","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584426916Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584435885Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"client":"0","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584445825Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"client":"1","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584455894Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"client":"2","":"downstairs","session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584466003Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584475443Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"fe30f370-f616-40df-bd12-0ce20117caa5 is now active with session: 82d1dd08-832a-403f-a52e-3743cec165b9","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584485742Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584495281Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584503171Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"session_id":"82d1dd08-832a-403f-a52e-3743cec165b9"} {"msg":"Activated sub_volume fe30f370-f616-40df-bd12-0ce20117caa5","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.5845126Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584521389Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.584859106Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"task":"crutest"} 01/30 Write block 509 len 12288 data: 12 12 10 02/30 Write block 314 len 12288 data: 17 15 18 03/30 Read block 520 len 20480 04/30 Write block 149 len 12288 data: 15 15 16 05/30 Read block 6 len 16384 06/30 Read block 186 len 16384 07/30 Flush 08/30 Write block 482 len 40960 data: 10 11 11 11 11 14 12 11 9 8 09/30 Write block 137 len 20480 data: 11 13 14 12 13 10/30 Read block 285 len 24576 11/30 Flush 12/30 Write block 335 len 24576 data: 12 12 10 10 10 11 13/30 Flush 14/30 Write block 376 len 32768 data: 11 12 15 13 12 12 13 12 15/30 Read block 120 len 28672 16/30 Read block 163 len 40960 17/30 Write block 126 len 28672 data: 12 8 9 8 8 9 11 18/30 Write block 324 len 20480 data: 14 10 9 10 12 19/30 Write block 188 len 36864 data: 10 9 9 10 11 11 11 12 9 20/30 Read block 307 len 40960 21/30 Flush 22/30 Write block 516 len 20480 data: 7 5 6 8 10 23/30 Read block 220 len 24576 24/30 Flush 25/30 Write block 509 len 20480 data: 13 13 11 6 9 26/30 Read block 157 len 32768 27/30 Read block 416 len 40960 28/30 Write block 452 len 8192 data: 8 8 29/30 Read block 522 len 12288 30/30 Write block 514 len 28672 data: 9 8 8 6 7 9 11 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 8 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Read 10 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 5 Done Sent Done false 1022 Acked Read 6 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Write 2 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 24 1 26 Done 29 6 29 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:14.593903601Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:14.59421091Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1632,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 210 210 210 F F F 1 020-039 122 122 122 220 220 220 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 123 123 123 225 225 225 F F F 6 120-139 124 123 124 229 225 229 F F F <--- 7 140-159 124 122 124 227 222 227 F T F <--- 8 160-179 121 121 121 219 219 219 F F F 9 180-199 124 116 124 229 212 229 F F F <--- 10 200-219 122 122 122 222 222 222 F F F 11 220-239 122 122 122 222 222 222 F F F 12 240-259 122 122 122 222 222 222 F F F 13 260-279 119 119 119 215 215 215 F F F 14 280-299 123 123 123 225 225 225 F F F 15 300-319 124 120 124 226 218 226 F T F <--- 16 320-339 124 121 124 229 219 229 F F F <--- 17 340-359 124 122 124 228 224 228 F F F <--- 18 360-379 124 115 124 229 210 229 F F F <--- 19 380-399 124 110 124 229 201 229 F F F <--- 20 400-419 123 123 123 225 225 225 F F F 21 420-439 116 116 116 212 212 212 F F F 22 440-459 120 120 120 216 216 216 T F T <--- 23 460-479 123 123 123 225 225 225 F F F 24 480-499 124 122 124 227 222 227 F F F <--- 25 500-519 124 121 124 230 219 230 T T T <--- 26 520-539 124 122 124 230 222 230 T F T <--- 27 540-559 122 122 122 222 222 222 F F F 28 560-579 116 116 116 211 211 211 F F F 29 580-599 113 113 113 206 206 206 F F F Max gen: 124, Max flush: 230 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 210 210 210 F F F 1 020-039 122 122 122 220 220 220 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 123 123 123 225 225 225 F F F 6 120-139 124 124 124 229 229 229 F F F 7 140-159 124 124 124 227 227 227 F F F 8 160-179 121 121 121 219 219 219 F F F 9 180-199 124 124 124 229 229 229 F F F 10 200-219 122 122 122 222 222 222 F F F 11 220-239 122 122 122 222 222 222 F F F 12 240-259 122 122 122 222 222 222 F F F 13 260-279 119 119 119 215 215 215 F F F 14 280-299 123 123 123 225 225 225 F F F 15 300-319 124 124 124 226 226 226 F F F 16 320-339 124 124 124 229 229 229 F F F 17 340-359 124 124 124 228 228 228 F F F 18 360-379 124 124 124 229 229 229 F F F 19 380-399 124 124 124 229 229 229 F F F 20 400-419 123 123 123 225 225 225 F F F 21 420-439 116 116 116 212 212 212 F F F 22 440-459 125 125 125 231 231 231 F F F 23 460-479 123 123 123 225 225 225 F F F 24 480-499 124 124 124 227 227 227 F F F 25 500-519 125 125 125 231 231 231 F F F 26 520-539 125 125 125 231 231 231 F F F 27 540-559 122 122 122 222 222 222 F F F 28 560-579 116 116 116 211 211 211 F F F 29 580-599 113 113 113 206 206 206 F F F Max gen: 125, Max flush: 231 Begin loop 63 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.836259715Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.844365014Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.844642205Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.844669033Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.844680192Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.844692162Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640} {"msg":"Crucible e36c1351-bdd9-462c-8318-4eb4b6ef4a27 has session id: acb0af9e-7518-4b49-8ae9-041e4f2199f4","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845032608Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"Upstairs opts: Upstairs UUID: e36c1351-bdd9-462c-8318-4eb4b6ef4a27, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845064426Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"Crucible stats registered with UUID: e36c1351-bdd9-462c-8318-4eb4b6ef4a27","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845075955Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:18.845085725Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845245414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845278772Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845290181Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845415672Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845440081Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640} {"msg":"e36c1351-bdd9-462c-8318-4eb4b6ef4a27 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845494737Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845505066Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"client":"0","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845515705Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"client":"1","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845528015Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"client":"2","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845538364Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845548803Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845559043Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:37628)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845623148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:46970)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845634477Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"ds_connection connected from Ok(127.0.0.1:35075)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.845645227Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846361958Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"client":"2","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846387606Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"client":"1","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846399655Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"client":"0","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846653358Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846695615Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"[0]R flush_numbers[0..12]: [210, 220, 211, 209, 204, 225, 229, 227, 219, 229, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846756171Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"downstairs","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"[0]R generation[0..12]: [115, 122, 116, 114, 112, 123, 124, 124, 121, 124, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.84676822Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"downstairs","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846782309Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"downstairs","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"[1]R flush_numbers[0..12]: [210, 220, 211, 209, 204, 225, 229, 227, 219, 229, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846798198Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"downstairs","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"[1]R generation[0..12]: [115, 122, 116, 114, 112, 123, 124, 124, 121, 124, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846809297Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"downstairs","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846820617Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"downstairs","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"[2]R flush_numbers[0..12]: [210, 220, 211, 209, 204, 225, 229, 227, 219, 229, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846831716Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"downstairs","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"[2]R generation[0..12]: [115, 122, 116, 114, 112, 123, 124, 124, 121, 124, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846842625Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"downstairs","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846853854Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"downstairs","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846863594Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"downstairs","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846873733Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"downstairs","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846883502Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"downstairs","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846893222Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"":"downstairs","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846903271Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.84691233Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"client":"0","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.84692245Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"client":"1","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846932479Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"client":"2","":"downstairs","session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846943928Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846953348Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"e36c1351-bdd9-462c-8318-4eb4b6ef4a27 is now active with session: acb0af9e-7518-4b49-8ae9-041e4f2199f4","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846963707Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846973156Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846981116Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"session_id":"acb0af9e-7518-4b49-8ae9-041e4f2199f4"} {"msg":"Activated sub_volume e36c1351-bdd9-462c-8318-4eb4b6ef4a27","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.846990685Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 name":"crucible","level":30,"time":"2026-07-28T04:43:18.847003254Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.847263817Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"task":"crutest"} 01/30 Write block 250 len 8192 data: 7 7 02/30 Write block 518 len 16384 data: 8 10 12 10 03/30 Write block 592 len 20480 data: 9 9 9 8 7 04/30 Read block 232 len 36864 05/30 Write block 33 len 24576 data: 13 11 10 11 9 10 06/30 Read block 182 len 16384 07/30 Flush 08/30 Write block 388 len 20480 data: 5 7 8 9 8 09/30 Read block 69 len 24576 10/30 Read block 502 len 28672 11/30 Flush 12/30 Write block 561 len 12288 data: 11 11 13 13/30 Read block 84 len 40960 14/30 Write block 0 len 24576 data: 4 6 6 6 8 10 15/30 Read block 538 len 12288 16/30 Read block 543 len 4096 17/30 Read block 60 len 24576 18/30 Write block 350 len 8192 data: 11 11 19/30 Flush 20/30 Write block 16 len 12288 data: 4 5 8 21/30 Read block 411 len 12288 22/30 Read block 569 len 4096 23/30 Read block 421 len 16384 24/30 Write block 527 len 16384 data: 12 11 11 11 25/30 Read block 9 len 32768 26/30 Read block 431 len 4096 27/30 Read block 426 len 32768 28/30 Read block 324 len 12288 29/30 Read block 32 len 32768 30/30 Write block 192 len 32768 data: 12 12 12 13 10 9 10 12 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 4 Done Sent Done false 1002 Acked Write 5 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Read 4 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Read 6 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:18.855783697Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:18.856068207Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1640,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 115 126 234 210 234 T F T <--- 1 020-039 126 122 126 232 220 232 F F F <--- 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 123 123 123 225 225 225 F F F 6 120-139 124 124 124 229 229 229 F F F 7 140-159 124 124 124 227 227 227 F F F 8 160-179 121 121 121 219 219 219 F F F 9 180-199 124 124 124 229 229 229 T F T <--- 10 200-219 122 122 122 222 222 222 F F F 11 220-239 122 122 122 222 222 222 F F F 12 240-259 126 122 126 232 222 232 F T F <--- 13 260-279 119 119 119 215 215 215 F F F 14 280-299 123 123 123 225 225 225 F F F 15 300-319 124 124 124 226 226 226 F F F 16 320-339 124 124 124 229 229 229 F F F 17 340-359 126 124 126 234 228 234 F F F <--- 18 360-379 124 124 124 229 229 229 F F F 19 380-399 126 124 126 233 229 233 F F F <--- 20 400-419 123 123 123 225 225 225 F F F 21 420-439 116 116 116 212 212 212 F F F 22 440-459 125 125 125 231 231 231 F F F 23 460-479 123 123 123 225 225 225 F F F 24 480-499 124 124 124 227 227 227 F F F 25 500-519 126 125 126 232 231 232 F F F <--- 26 520-539 126 125 126 232 231 232 T F T <--- 27 540-559 122 122 122 222 222 222 F F F 28 560-579 126 116 126 234 211 234 F F F <--- 29 580-599 126 113 126 232 206 232 F F F <--- Max gen: 126, Max flush: 234 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 235 235 235 F F F 1 020-039 126 126 126 232 232 232 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 123 123 123 225 225 225 F F F 6 120-139 124 124 124 229 229 229 F F F 7 140-159 124 124 124 227 227 227 F F F 8 160-179 121 121 121 219 219 219 F F F 9 180-199 127 127 127 235 235 235 F F F 10 200-219 122 122 122 222 222 222 F F F 11 220-239 122 122 122 222 222 222 F F F 12 240-259 126 126 126 232 232 232 F F F 13 260-279 119 119 119 215 215 215 F F F 14 280-299 123 123 123 225 225 225 F F F 15 300-319 124 124 124 226 226 226 F F F 16 320-339 124 124 124 229 229 229 F F F 17 340-359 126 126 126 234 234 234 F F F 18 360-379 124 124 124 229 229 229 F F F 19 380-399 126 126 126 233 233 233 F F F 20 400-419 123 123 123 225 225 225 F F F 21 420-439 116 116 116 212 212 212 F F F 22 440-459 125 125 125 231 231 231 F F F 23 460-479 123 123 123 225 225 225 F F F 24 480-499 124 124 124 227 227 227 F F F 25 500-519 126 126 126 232 232 232 F F F 26 520-539 127 127 127 235 235 235 F F F 27 540-559 122 122 122 222 222 222 F F F 28 560-579 126 126 126 234 234 234 F F F 29 580-599 126 126 126 232 232 232 F F F Max gen: 127, Max flush: 235 Begin loop 64 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.091040174Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.099170391Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.099451382Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.09947926Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.099488899Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.099501019Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648} {"msg":"Crucible 69e7cc42-c164-45d7-b92e-98534b7958a0 has session id: 612b0d28-4e04-4060-82fa-0ff595254d34","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.099812317Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"Upstairs opts: Upstairs UUID: 69e7cc42-c164-45d7-b92e-98534b7958a0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.099842015Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"Crucible stats registered with UUID: 69e7cc42-c164-45d7-b92e-98534b7958a0","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.099854935Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:23.099864584Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.100031773Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.10006324Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.10007467Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.100168953Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.100194371Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648} {"msg":"69e7cc42-c164-45d7-b92e-98534b7958a0 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.100206711Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.10021604Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"client":"0","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.100226089Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"client":"1","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.100236239Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"client":"2","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.100246238Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.100257937Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.100268406Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"ds_connection connected from Ok(127.0.0.1:33765)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.100295845Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"ds_connection connected from Ok(127.0.0.1:44632)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.100309164Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"ds_connection connected from Ok(127.0.0.1:46353)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.100323683Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.100958099Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"client":"2","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101015596Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"client":"0","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101030075Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"client":"1","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.10123862Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101264999Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"[0]R flush_numbers[0..12]: [235, 232, 211, 209, 204, 225, 229, 227, 219, 235, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101352903Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"downstairs","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"[0]R generation[0..12]: [127, 126, 116, 114, 112, 123, 124, 124, 121, 127, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101364632Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"downstairs","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101376041Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"downstairs","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"[1]R flush_numbers[0..12]: [235, 232, 211, 209, 204, 225, 229, 227, 219, 235, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.10138707Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"downstairs","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"[1]R generation[0..12]: [127, 126, 116, 114, 112, 123, 124, 124, 121, 127, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.10139801Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"downstairs","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101409359Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"downstairs","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"[2]R flush_numbers[0..12]: [235, 232, 211, 209, 204, 225, 229, 227, 219, 235, 222, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101420398Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"downstairs","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"[2]R generation[0..12]: [127, 126, 116, 114, 112, 123, 124, 124, 121, 127, 122, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101433507Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"downstairs","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101444766Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"downstairs","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101454706Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"downstairs","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101464855Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"downstairs","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101474604Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"downstairs","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101484554Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"":"downstairs","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101494653Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101503782Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"client":"0","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101513792Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"client":"1","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101523771Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"client":"2","":"downstairs","session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.10153394Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.10154335Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"69e7cc42-c164-45d7-b92e-98534b7958a0 is now active with session: 612b0d28-4e04-4060-82fa-0ff595254d34","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101553749Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101563368Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101571238Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"session_id":"612b0d28-4e04-4060-82fa-0ff595254d34"} {"msg":"Activated sub_volume 69e7cc42-c164-45d7-b92e-98534b7958a0","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101580797Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101589237Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.101916374Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"task":"crutest"} 01/30 Write block 572 len 20480 data: 12 14 12 15 14 02/30 Read block 212 len 16384 03/30 Read block 329 len 16384 04/30 Write block 339 len 20480 data: 11 12 11 11 11 05/30 Write block 293 len 24576 data: 7 8 8 9 11 14 06/30 Write block 574 len 4096 data: 13 07/30 Write block 19 len 40960 data: 9 10 10 11 10 10 9 9 8 10 08/30 Read block 87 len 24576 09/30 Write block 539 len 28672 data: 15 12 10 12 10 11 11 10/30 Write block 150 len 20480 data: 16 17 12 9 9 11/30 Write block 543 len 32768 data: 11 12 12 11 11 12 12 10 12/30 Read block 275 len 16384 13/30 Read block 94 len 12288 14/30 Write block 347 len 12288 data: 6 5 10 15/30 Read block 359 len 20480 16/30 Write block 463 len 20480 data: 11 12 16 17 16 17/30 Write block 204 len 8192 data: 12 13 18/30 Write block 240 len 36864 data: 10 12 14 13 12 11 14 10 7 19/30 Write block 344 len 8192 data: 12 11 20/30 Read block 405 len 12288 21/30 Write block 318 len 8192 data: 19 18 22/30 Write block 569 len 16384 data: 11 12 11 13 23/30 Read block 574 len 20480 24/30 Read block 470 len 12288 25/30 Write block 450 len 12288 data: 9 9 9 26/30 Write block 7 len 32768 data: 13 12 12 9 9 9 10 7 27/30 Write block 342 len 4096 data: 12 28/30 Write block 449 len 4096 data: 9 29/30 Read block 216 len 4096 30/30 Read block 512 len 12288 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 4 Done Sent Done false 1002 Acked Read 4 Done Sent Done false 1003 Acked Write 5 Done Sent Done false 1004 Acked Write 6 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Write 3 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 0 29 Done 30 1 30 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:23.108955355Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:23.109254525Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1648,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 235 235 235 T F T <--- 1 020-039 126 126 126 232 232 232 T F T <--- 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 123 123 123 225 225 225 F F F 6 120-139 124 124 124 229 229 229 F F F 7 140-159 124 124 124 227 227 227 T F T <--- 8 160-179 121 121 121 219 219 219 F F F 9 180-199 127 127 127 235 235 235 F F F 10 200-219 122 122 122 222 222 222 T F T <--- 11 220-239 122 122 122 222 222 222 F F F 12 240-259 126 126 126 232 232 232 T F T <--- 13 260-279 119 119 119 215 215 215 F F F 14 280-299 123 123 123 225 225 225 T F T <--- 15 300-319 124 124 124 226 226 226 T F T <--- 16 320-339 124 124 124 229 229 229 T F T <--- 17 340-359 126 126 126 234 234 234 T F T <--- 18 360-379 124 124 124 229 229 229 F F F 19 380-399 126 126 126 233 233 233 F F F 20 400-419 123 123 123 225 225 225 F F F 21 420-439 116 116 116 212 212 212 F F F 22 440-459 125 125 125 231 231 231 T F T <--- 23 460-479 123 123 123 225 225 225 T F T <--- 24 480-499 124 124 124 227 227 227 F F F 25 500-519 126 126 126 232 232 232 F F F 26 520-539 127 127 127 235 235 235 T F T <--- 27 540-559 122 122 122 222 222 222 T F T <--- 28 560-579 126 126 126 234 234 234 T T T 29 580-599 126 126 126 232 232 232 F F F Max gen: 127, Max flush: 235 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 236 236 236 F F F 1 020-039 128 128 128 236 236 236 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 F F F 4 080-099 112 112 112 204 204 204 F F F 5 100-119 123 123 123 225 225 225 F F F 6 120-139 124 124 124 229 229 229 F F F 7 140-159 128 128 128 236 236 236 F F F 8 160-179 121 121 121 219 219 219 F F F 9 180-199 127 127 127 235 235 235 F F F 10 200-219 128 128 128 236 236 236 F F F 11 220-239 122 122 122 222 222 222 F F F 12 240-259 128 128 128 236 236 236 F F F 13 260-279 119 119 119 215 215 215 F F F 14 280-299 128 128 128 236 236 236 F F F 15 300-319 128 128 128 236 236 236 F F F 16 320-339 128 128 128 236 236 236 F F F 17 340-359 128 128 128 236 236 236 F F F 18 360-379 124 124 124 229 229 229 F F F 19 380-399 126 126 126 233 233 233 F F F 20 400-419 123 123 123 225 225 225 F F F 21 420-439 116 116 116 212 212 212 F F F 22 440-459 128 128 128 236 236 236 F F F 23 460-479 128 128 128 236 236 236 F F F 24 480-499 124 124 124 227 227 227 F F F 25 500-519 126 126 126 232 232 232 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 128 128 128 236 236 236 F F F 28 560-579 128 128 128 236 236 236 F F F 29 580-599 126 126 126 232 232 232 F F F Max gen: 128, Max flush: 236 Begin loop 65 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.359134787Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.367587752Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.367883652Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.36790879Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.367927239Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.367939488Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656} {"msg":"Crucible dd2b3ebe-1f50-4207-a1f7-44d7c3d2e633 has session id: 2e3586c7-54a9-47e7-9963-7a6931ef4160","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368266726Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"Upstairs opts: Upstairs UUID: dd2b3ebe-1f50-4207-a1f7-44d7c3d2e633, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368287485Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"Crucible stats registered with UUID: dd2b3ebe-1f50-4207-a1f7-44d7c3d2e633","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368297814Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:27.368313493Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.36849953Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368531378Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368544987Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.36864904Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368675168Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656} {"msg":"dd2b3ebe-1f50-4207-a1f7-44d7c3d2e633 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368699287Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368708906Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"client":"0","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368718935Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"client":"1","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368729055Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"client":"2","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368739164Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368749493Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368759953Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"ds_connection connected from Ok(127.0.0.1:44187)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368834717Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"ds_connection connected from Ok(127.0.0.1:39818)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.368995996Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"ds_connection connected from Ok(127.0.0.1:51850)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369024385Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369429157Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"client":"0","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.36953163Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"client":"2","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369543779Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"client":"1","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369723087Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369810981Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"[0]R flush_numbers[0..12]: [236, 236, 211, 209, 204, 225, 229, 236, 219, 235, 236, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369868457Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"downstairs","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"[0]R generation[0..12]: [128, 128, 116, 114, 112, 123, 124, 128, 121, 127, 128, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369879956Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"downstairs","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369891286Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"downstairs","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"[1]R flush_numbers[0..12]: [236, 236, 211, 209, 204, 225, 229, 236, 219, 235, 236, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369902335Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"downstairs","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"[1]R generation[0..12]: [128, 128, 116, 114, 112, 123, 124, 128, 121, 127, 128, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369913164Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"downstairs","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369924133Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"downstairs","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"[2]R flush_numbers[0..12]: [236, 236, 211, 209, 204, 225, 229, 236, 219, 235, 236, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369934903Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"downstairs","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"[2]R generation[0..12]: [128, 128, 116, 114, 112, 123, 124, 128, 121, 127, 128, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369945692Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"downstairs","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369957001Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"downstairs","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"Max found gen is 129","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.36996668Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"downstairs","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"Generation requested: 130 >= found:129","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.36997693Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"downstairs","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369986669Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"downstairs","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.369997708Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"":"downstairs","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.370007678Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.370016877Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"client":"0","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.370026916Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"client":"1","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.370036866Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"client":"2","":"downstairs","session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.370046845Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.370056294Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"dd2b3ebe-1f50-4207-a1f7-44d7c3d2e633 is now active with session: 2e3586c7-54a9-47e7-9963-7a6931ef4160","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.370066634Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.370076083Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.370084052Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"session_id":"2e3586c7-54a9-47e7-9963-7a6931ef4160"} {"msg":"Activated sub_volume dd2b3ebe-1f50-4207-a1f7-44d7c3d2e633","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.370093542Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.370102081Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.370435698Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"task":"crutest"} 01/30 Write block 92 len 8192 data: 11 14 02/30 Write block 97 len 24576 data: 11 13 12 8 8 7 03/30 Write block 417 len 16384 data: 11 10 10 11 04/30 Write block 86 len 36864 data: 5 5 6 7 8 9 12 15 13 05/30 Read block 48 len 32768 06/30 Write block 343 len 8192 data: 12 13 07/30 Read block 324 len 24576 08/30 Flush 09/30 Write block 467 len 28672 data: 17 16 15 15 13 11 10 10/30 Write block 308 len 16384 data: 12 13 13 16 11/30 Write block 357 len 28672 data: 10 10 13 13 16 11 10 12/30 Read block 229 len 12288 13/30 Read block 242 len 4096 14/30 Write block 336 len 28672 data: 13 11 11 12 13 12 13 15/30 Read block 264 len 20480 16/30 Flush 17/30 Write block 293 len 36864 data: 8 9 9 10 12 15 13 12 10 18/30 Write block 388 len 32768 data: 6 8 9 10 9 8 12 12 19/30 Write block 279 len 28672 data: 6 6 7 7 5 5 4 20/30 Read block 577 len 12288 21/30 Write block 68 len 12288 data: 4 6 4 22/30 Read block 364 len 40960 23/30 Read block 569 len 36864 24/30 Read block 349 len 28672 25/30 Write block 541 len 4096 data: 11 26/30 Write block 35 len 8192 data: 11 12 27/30 Write block 468 len 16384 data: 17 16 16 14 28/30 Write block 152 len 4096 data: 13 29/30 Read block 469 len 12288 30/30 Read block 31 len 20480 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 8 Done Sent Done false 1005 Acked Write 2 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Write 8 Done Sent Done false 1018 Acked Write 7 Done Sent Done false 1019 Acked Read 3 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 3 Done Sent Done false 1029 Acked Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 0 26 Done 30 4 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:27.378111236Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:27.378404296Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1656,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 236 236 236 F F F 1 020-039 128 128 128 236 236 236 T F T <--- 2 040-059 116 116 116 211 211 211 F F F 3 060-079 114 114 114 209 209 209 T F T <--- 4 080-099 130 112 130 237 204 237 F T F <--- 5 100-119 130 123 130 237 225 237 F T F <--- 6 120-139 124 124 124 229 229 229 F F F 7 140-159 128 128 128 236 236 236 T F T <--- 8 160-179 121 121 121 219 219 219 F F F 9 180-199 127 127 127 235 235 235 F F F 10 200-219 128 128 128 236 236 236 F F F 11 220-239 122 122 122 222 222 222 F F F 12 240-259 128 128 128 236 236 236 F F F 13 260-279 119 119 119 215 215 215 T F T <--- 14 280-299 128 128 128 236 236 236 T F T <--- 15 300-319 130 128 130 238 236 238 T F T <--- 16 320-339 130 128 130 238 236 238 F F F <--- 17 340-359 130 128 130 238 236 238 F F F <--- 18 360-379 130 124 130 238 229 238 F F F <--- 19 380-399 126 126 126 233 233 233 T F T <--- 20 400-419 130 123 130 237 225 237 F T F <--- 21 420-439 130 116 130 237 212 237 F T F <--- 22 440-459 128 128 128 236 236 236 F F F 23 460-479 130 128 130 238 236 238 T F T <--- 24 480-499 124 124 124 227 227 227 F F F 25 500-519 126 126 126 232 232 232 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 128 128 128 236 236 236 T F T <--- 28 560-579 128 128 128 236 236 236 F F F 29 580-599 126 126 126 232 232 232 F F F Max gen: 130, Max flush: 238 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 236 236 236 F F F 1 020-039 131 131 131 239 239 239 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 131 131 131 239 239 239 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 130 130 130 237 237 237 F F F 6 120-139 124 124 124 229 229 229 F F F 7 140-159 131 131 131 239 239 239 F F F 8 160-179 121 121 121 219 219 219 F F F 9 180-199 127 127 127 235 235 235 F F F 10 200-219 128 128 128 236 236 236 F F F 11 220-239 122 122 122 222 222 222 F F F 12 240-259 128 128 128 236 236 236 F F F 13 260-279 131 131 131 239 239 239 F F F 14 280-299 131 131 131 239 239 239 F F F 15 300-319 131 131 131 239 239 239 F F F 16 320-339 130 130 130 238 238 238 F F F 17 340-359 130 130 130 238 238 238 F F F 18 360-379 130 130 130 238 238 238 F F F 19 380-399 131 131 131 239 239 239 F F F 20 400-419 130 130 130 237 237 237 F F F 21 420-439 130 130 130 237 237 237 F F F 22 440-459 128 128 128 236 236 236 F F F 23 460-479 131 131 131 239 239 239 F F F 24 480-499 124 124 124 227 227 227 F F F 25 500-519 126 126 126 232 232 232 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 131 131 131 239 239 239 F F F 28 560-579 128 128 128 236 236 236 F F F 29 580-599 126 126 126 232 232 232 F F F Max gen: 131, Max flush: 239 Begin loop 66 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.632461655Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:31.639557182Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.639622937Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.646593903Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.646842386Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.646865255Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.646873934Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.646885923Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664} {"msg":"Crucible b4f3ea0e-dc82-4b9d-9a73-3b6dd941b326 has session id: ba6420b6-66ff-425a-b60e-d30da61882c5","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647275677Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"Upstairs opts: Upstairs UUID: b4f3ea0e-dc82-4b9d-9a73-3b6dd941b326, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647306125Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"Crucible stats registered with UUID: b4f3ea0e-dc82-4b9d-9a73-3b6dd941b326","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647317374Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:31.647327193Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647480543Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647497332Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647507771Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647638502Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.64766498Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664} {"msg":"b4f3ea0e-dc82-4b9d-9a73-3b6dd941b326 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647690338Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647702108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"client":"0","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647712247Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"client":"1","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647722096Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"client":"2","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647732006Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647742395Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647752694Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35871)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.64780728Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41405)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.64781865Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46758)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.647921553Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.64854254Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"client":"1","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.648562429Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"client":"2","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.648574108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"client":"0","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.648872908Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.648882847Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"[0]R flush_numbers[0..12]: [236, 239, 211, 239, 237, 237, 229, 239, 219, 235, 236, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.648926534Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"downstairs","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"[0]R generation[0..12]: [128, 131, 116, 131, 130, 130, 124, 131, 121, 127, 128, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.648939513Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"downstairs","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.648950873Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"downstairs","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"[1]R flush_numbers[0..12]: [236, 239, 211, 239, 237, 237, 229, 239, 219, 235, 236, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.648961892Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"downstairs","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"[1]R generation[0..12]: [128, 131, 116, 131, 130, 130, 124, 131, 121, 127, 128, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.648972801Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"downstairs","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.64898418Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"downstairs","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"[2]R flush_numbers[0..12]: [236, 239, 211, 239, 237, 237, 229, 239, 219, 235, 236, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.64899526Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"downstairs","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"[2]R generation[0..12]: [128, 131, 116, 131, 130, 130, 124, 131, 121, 127, 128, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649005959Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"downstairs","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649017298Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"downstairs","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649027107Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"downstairs","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649037257Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"downstairs","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649047086Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"downstairs","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649056875Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"":"downstairs","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649067075Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649076154Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"client":"0","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649087333Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"client":"1","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649097383Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"client":"2","":"downstairs","session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649107432Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649116921Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"b4f3ea0e-dc82-4b9d-9a73-3b6dd941b326 is now active with session: ba6420b6-66ff-425a-b60e-d30da61882c5","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649127241Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.6491366Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649144909Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"session_id":"ba6420b6-66ff-425a-b60e-d30da61882c5"} {"msg":"Activated sub_volume b4f3ea0e-dc82-4b9d-9a73-3b6dd941b326","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649154409Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649162888Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.649485806Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"task":"crutest"} 01/30 Write block 75 len 4096 data: 8 02/30 Read block 293 len 36864 03/30 Write block 250 len 16384 data: 8 8 6 6 04/30 Read block 64 len 12288 05/30 Write block 444 len 8192 data: 7 6 06/30 Read block 554 len 32768 07/30 Read block 155 len 8192 08/30 Write block 349 len 32768 data: 11 12 12 10 9 9 11 11 09/30 Read block 178 len 32768 10/30 Write block 457 len 32768 data: 11 14 12 11 10 8 12 13 11/30 Read block 315 len 32768 12/30 Write block 317 len 8192 data: 18 20 13/30 Write block 288 len 40960 data: 5 4 3 4 6 9 10 10 11 13 14/30 Read block 185 len 36864 15/30 Read block 536 len 16384 16/30 Write block 171 len 4096 data: 12 17/30 Read block 122 len 16384 18/30 Write block 481 len 8192 data: 9 11 19/30 Read block 268 len 12288 20/30 Read block 254 len 32768 21/30 Read block 96 len 32768 22/30 Read block 191 len 8192 23/30 Write block 547 len 36864 data: 12 13 13 11 9 9 9 8 11 24/30 Read block 406 len 24576 25/30 Write block 280 len 4096 data: 7 26/30 Read block 404 len 36864 27/30 Read block 106 len 20480 28/30 Read block 194 len 16384 29/30 Write block 185 len 24576 data: 10 8 7 11 10 10 30/30 Read block 280 len 32768 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 2 Done Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 2 Sent Done Done false 1007 Acked Write 8 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Write 10 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Read 4 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Write 2 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Read 2 Sent Done Done false 1022 Acked Write 9 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Write 1 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 1 0 26 Done 5 29 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:31.658204403Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:31.658497123Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1664,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 236 236 236 F F F 1 020-039 131 131 131 239 239 239 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 131 131 131 239 239 239 T T T 4 080-099 130 130 130 237 237 237 F F F 5 100-119 130 130 130 237 237 237 F F F 6 120-139 124 124 124 229 229 229 F F F 7 140-159 131 131 131 239 239 239 F F F 8 160-179 121 121 121 219 219 219 F T T <--- 9 180-199 127 127 127 235 235 235 F T T <--- 10 200-219 128 128 128 236 236 236 F F F 11 220-239 122 122 122 222 222 222 F F F 12 240-259 128 128 128 236 236 236 T T T 13 260-279 131 131 131 239 239 239 F F F 14 280-299 131 131 131 239 239 239 F T T <--- 15 300-319 131 131 131 239 239 239 F T T <--- 16 320-339 130 130 130 238 238 238 F F F 17 340-359 130 130 130 238 238 238 F T T <--- 18 360-379 130 130 130 238 238 238 F F F 19 380-399 131 131 131 239 239 239 F F F 20 400-419 130 130 130 237 237 237 F F F 21 420-439 130 130 130 237 237 237 F F F 22 440-459 128 128 128 236 236 236 T T T 23 460-479 131 131 131 239 239 239 F T T <--- 24 480-499 124 124 124 227 227 227 F T T <--- 25 500-519 126 126 126 232 232 232 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 131 131 131 239 239 239 F T T <--- 28 560-579 128 128 128 236 236 236 F F F 29 580-599 126 126 126 232 232 232 F F F Max gen: 131, Max flush: 239 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 66 Verifying data now Files /tmp/test_repair/test_repair_verify.data and /tmp/test_repair/test_repair_verify.data.last differ 460,461d459 < 10, < 13, 462a461,462 > 14, > 12, diff found after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 236 236 236 F F F 1 020-039 131 131 131 239 239 239 F F F 2 040-059 116 116 116 211 211 211 F F F 3 060-079 132 132 132 240 240 240 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 130 130 130 237 237 237 F F F 6 120-139 124 124 124 229 229 229 F F F 7 140-159 131 131 131 239 239 239 F F F 8 160-179 132 132 132 240 240 240 F F F 9 180-199 132 132 132 240 240 240 F F F 10 200-219 128 128 128 236 236 236 F F F 11 220-239 122 122 122 222 222 222 F F F 12 240-259 132 132 132 240 240 240 F F F 13 260-279 131 131 131 239 239 239 F F F 14 280-299 132 132 132 240 240 240 F F F 15 300-319 132 132 132 240 240 240 F F F 16 320-339 130 130 130 238 238 238 F F F 17 340-359 132 132 132 240 240 240 F F F 18 360-379 130 130 130 238 238 238 F F F 19 380-399 131 131 131 239 239 239 F F F 20 400-419 130 130 130 237 237 237 F F F 21 420-439 130 130 130 237 237 237 F F F 22 440-459 132 132 132 240 240 240 F F F 23 460-479 132 132 132 240 240 240 F F F 24 480-499 132 132 132 240 240 240 F F F 25 500-519 126 126 126 232 232 232 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 132 132 132 240 240 240 F F F 28 560-579 128 128 128 236 236 236 F F F 29 580-599 126 126 126 232 232 232 F F F Max gen: 132, Max flush: 240 Begin loop 67 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.913923977Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.922063213Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.922343684Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.922369962Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.922378722Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.922390401Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675} {"msg":"Crucible 6aa2882c-0fb7-4f99-936e-cd1eec11dc5f has session id: 69bdb39e-aafa-45a3-9891-c15ea4d7bb05","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.922710109Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"Upstairs opts: Upstairs UUID: 6aa2882c-0fb7-4f99-936e-cd1eec11dc5f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.922744197Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"Crucible stats registered with UUID: 6aa2882c-0fb7-4f99-936e-cd1eec11dc5f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.922758546Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:35.922788044Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.922965372Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.92299641Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.923007769Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.923125171Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.92313898Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675} {"msg":"6aa2882c-0fb7-4f99-936e-cd1eec11dc5f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.923150069Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.923159339Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"client":"0","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.923169508Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"client":"1","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.923179477Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"client":"2","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.923189567Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.923199876Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.923212295Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"ds_connection connected from Ok(127.0.0.1:53024)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.92328064Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"io task","client":"2","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"ds_connection connected from Ok(127.0.0.1:33268)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.923297459Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"io task","client":"1","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"ds_connection connected from Ok(127.0.0.1:36328)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.923337677Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"io task","client":"0","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.923943535Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"client":"1","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924011351Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"client":"0","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.92402379Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"client":"2","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924237725Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924281982Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"[0]R flush_numbers[0..12]: [236, 239, 211, 240, 237, 237, 229, 239, 240, 240, 236, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924346828Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"downstairs","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"[0]R generation[0..12]: [128, 131, 116, 132, 130, 130, 124, 131, 132, 132, 128, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924358647Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"downstairs","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924370026Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"downstairs","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"[1]R flush_numbers[0..12]: [236, 239, 211, 240, 237, 237, 229, 239, 240, 240, 236, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924380976Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"downstairs","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"[1]R generation[0..12]: [128, 131, 116, 132, 130, 130, 124, 131, 132, 132, 128, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924391755Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"downstairs","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924405414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"downstairs","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"[2]R flush_numbers[0..12]: [236, 239, 211, 240, 237, 237, 229, 239, 240, 240, 236, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924416443Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"downstairs","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"[2]R generation[0..12]: [128, 131, 116, 132, 130, 130, 124, 131, 132, 132, 128, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924427312Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"downstairs","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924438542Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"downstairs","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"Max found gen is 133","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924448441Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"downstairs","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"Generation requested: 134 >= found:133","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.92445857Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"downstairs","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.92446842Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"downstairs","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924478229Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"":"downstairs","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924488368Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924497538Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"client":"0","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924507537Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"client":"1","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924517526Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"client":"2","":"downstairs","session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924527586Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924536945Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"6aa2882c-0fb7-4f99-936e-cd1eec11dc5f is now active with session: 69bdb39e-aafa-45a3-9891-c15ea4d7bb05","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924547144Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924557394Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"session_id":"69bdb39e-aafa-45a3-9891-c15ea4d7bb05"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924566793Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675} {"msg":"Activated sub_volume 6aa2882c-0fb7-4f99-936e-cd1eec11dc5f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924575092Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924583722Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.924912709Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"task":"crutest"} 01/30 Write block 156 len 4096 data: 9 02/30 Write block 129 len 4096 data: 9 03/30 Flush 04/30 Write block 499 len 36864 data: 9 10 10 10 10 12 14 10 13 05/30 Read block 307 len 4096 06/30 Read block 169 len 36864 07/30 Write block 251 len 16384 data: 9 7 7 9 08/30 Flush 09/30 Write block 204 len 32768 data: 13 14 13 11 14 11 12 13 10/30 Write block 21 len 32768 data: 11 12 11 11 10 10 9 11 11/30 Write block 446 len 4096 data: 4 12/30 Write block 547 len 28672 data: 13 14 14 12 10 10 10 13/30 Write block 428 len 8192 data: 12 10 14/30 Write block 367 len 32768 data: 10 10 11 10 9 12 11 12 15/30 Read block 71 len 12288 16/30 Read block 406 len 24576 17/30 Write block 174 len 24576 data: 13 10 7 5 6 7 18/30 Read block 303 len 24576 19/30 Write block 44 len 40960 data: 10 10 10 9 8 7 8 9 7 5 20/30 Read block 502 len 12288 21/30 Write block 365 len 40960 data: 12 13 11 11 12 11 10 13 12 13 22/30 Read block 291 len 32768 23/30 Read block 29 len 8192 24/30 Read block 106 len 36864 25/30 Write block 48 len 28672 data: 9 8 9 10 8 6 7 26/30 Read block 56 len 32768 27/30 Read block 277 len 12288 28/30 Read block 281 len 24576 29/30 Write block 556 len 20480 data: 9 8 9 12 11 30/30 Read block 48 len 28672 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Sent false 1001 Acked Write 1 Done Done Sent false 1002 Acked Flush 0 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Read 1 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Write 4 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 8 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 8 Done Done Sent false 1022 Acked Read 2 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Write 5 Done Sent Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 30 32 Done 30 28 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:35.93284191Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:35.933150399Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1675,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 236 236 236 F F F 1 020-039 131 131 131 239 239 239 T T F <--- 2 040-059 116 116 116 211 211 211 T T F <--- 3 060-079 132 132 132 240 240 240 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 130 130 130 237 237 237 F F F 6 120-139 134 134 124 241 241 229 F F F <--- 7 140-159 134 134 131 241 241 239 F F F <--- 8 160-179 132 132 132 240 240 240 T T F <--- 9 180-199 132 132 132 240 240 240 F F F 10 200-219 128 128 128 236 236 236 T T F <--- 11 220-239 122 122 122 222 222 222 F F F 12 240-259 134 134 132 242 242 240 F F F <--- 13 260-279 131 131 131 239 239 239 F F F 14 280-299 132 132 132 240 240 240 F F F 15 300-319 132 132 132 240 240 240 F F F 16 320-339 130 130 130 238 238 238 F F F 17 340-359 132 132 132 240 240 240 F F F 18 360-379 130 130 130 238 238 238 T T F <--- 19 380-399 131 131 131 239 239 239 F F F 20 400-419 130 130 130 237 237 237 F F F 21 420-439 130 130 130 237 237 237 T T F <--- 22 440-459 132 132 132 240 240 240 T T F <--- 23 460-479 132 132 132 240 240 240 F F F 24 480-499 134 134 132 242 242 240 F F F <--- 25 500-519 134 134 126 242 242 232 F F F <--- 26 520-539 128 128 128 236 236 236 F F F 27 540-559 132 132 132 240 240 240 T T F <--- 28 560-579 128 128 128 236 236 236 T T F <--- 29 580-599 126 126 126 232 232 232 F F F Max gen: 134, Max flush: 242 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 236 236 236 F F F 1 020-039 135 135 135 243 243 243 F F F 2 040-059 135 135 135 243 243 243 F F F 3 060-079 132 132 132 240 240 240 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 130 130 130 237 237 237 F F F 6 120-139 134 134 134 241 241 241 F F F 7 140-159 134 134 134 241 241 241 F F F 8 160-179 135 135 135 243 243 243 F F F 9 180-199 132 132 132 240 240 240 F F F 10 200-219 135 135 135 243 243 243 F F F 11 220-239 122 122 122 222 222 222 F F F 12 240-259 134 134 134 242 242 242 F F F 13 260-279 131 131 131 239 239 239 F F F 14 280-299 132 132 132 240 240 240 F F F 15 300-319 132 132 132 240 240 240 F F F 16 320-339 130 130 130 238 238 238 F F F 17 340-359 132 132 132 240 240 240 F F F 18 360-379 135 135 135 243 243 243 F F F 19 380-399 131 131 131 239 239 239 F F F 20 400-419 130 130 130 237 237 237 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 135 135 135 243 243 243 F F F 23 460-479 132 132 132 240 240 240 F F F 24 480-499 134 134 134 242 242 242 F F F 25 500-519 134 134 134 242 242 242 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 135 135 135 243 243 243 F F F 28 560-579 135 135 135 243 243 243 F F F 29 580-599 126 126 126 232 232 232 F F F Max gen: 135, Max flush: 243 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.177011291Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:40.184268927Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.184334743Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.191095783Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.191350525Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.191385553Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.191395102Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.191407371Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683} {"msg":"Crucible d4e916ca-281e-43b2-b615-51d54258db5f has session id: cade8a58-95d7-487e-ab2a-f318db3a4425","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.191744248Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"Upstairs opts: Upstairs UUID: d4e916ca-281e-43b2-b615-51d54258db5f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.191774076Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"Crucible stats registered with UUID: d4e916ca-281e-43b2-b615-51d54258db5f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.191788825Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:40.191798535Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.191974733Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.192005681Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.19201694Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.192151891Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.192171259Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683} {"msg":"d4e916ca-281e-43b2-b615-51d54258db5f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.192207757Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.192217676Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"client":"0","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.192227706Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"client":"1","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.192237675Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"client":"2","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.192247594Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.192257894Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.192268113Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"ds_connection connected from Ok(127.0.0.1:49387)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.192365146Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"io task","client":"1","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"ds_connection connected from Ok(127.0.0.1:56057)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.192486618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"io task","client":"2","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"ds_connection connected from Ok(127.0.0.1:49511)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.192510786Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"io task","client":"0","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193021022Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"client":"2","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.19303876Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"client":"1","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193118875Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"client":"0","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193357549Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193395756Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"[0]R flush_numbers[0..12]: [236, 243, 243, 240, 237, 237, 241, 241, 243, 240, 243, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193453052Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"downstairs","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"[0]R generation[0..12]: [128, 135, 135, 132, 130, 130, 134, 134, 135, 132, 135, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193464651Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"downstairs","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193475891Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"downstairs","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"[1]R flush_numbers[0..12]: [236, 243, 243, 240, 237, 237, 241, 241, 243, 240, 243, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.19348683Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"downstairs","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"[1]R generation[0..12]: [128, 135, 135, 132, 130, 130, 134, 134, 135, 132, 135, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193497709Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"downstairs","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193508968Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"downstairs","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"[2]R flush_numbers[0..12]: [236, 243, 243, 240, 237, 237, 241, 241, 243, 240, 243, 222]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193519888Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"downstairs","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"[2]R generation[0..12]: [128, 135, 135, 132, 130, 130, 134, 134, 135, 132, 135, 122]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193530777Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"downstairs","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193543736Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"downstairs","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193553595Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"downstairs","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193563615Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"downstairs","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193573264Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"downstairs","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193582923Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"":"downstairs","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193593313Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193602592Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"client":"0","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193613051Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"client":"1","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193623281Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"client":"2","":"downstairs","session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.19363384Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193643439Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"d4e916ca-281e-43b2-b615-51d54258db5f is now active with session: cade8a58-95d7-487e-ab2a-f318db3a4425","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193653849Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193663318Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193671267Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"session_id":"cade8a58-95d7-487e-ab2a-f318db3a4425"} {"msg":"Activated sub_volume d4e916ca-281e-43b2-b615-51d54258db5f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193680727Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193689126Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.193986396Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"task":"crutest"} 01/30 Write block 131 len 8192 data: 10 12 02/30 Read block 119 len 8192 03/30 Read block 2 len 40960 04/30 Read block 98 len 28672 05/30 Read block 132 len 20480 06/30 Read block 351 len 12288 07/30 Write block 136 len 28672 data: 10 12 14 15 13 14 14 08/30 Write block 208 len 40960 data: 15 12 13 14 12 11 9 12 13 11 09/30 Read block 122 len 32768 10/30 Read block 201 len 36864 11/30 Read block 228 len 20480 12/30 Write block 410 len 12288 data: 7 10 10 13/30 Read block 479 len 4096 14/30 Write block 540 len 20480 data: 13 12 13 12 13 15/30 Read block 231 len 32768 16/30 Read block 574 len 4096 17/30 Read block 31 len 16384 18/30 Write block 28 len 28672 data: 12 9 11 13 10 14 12 19/30 Read block 110 len 16384 20/30 Read block 318 len 36864 21/30 Flush 22/30 Write block 553 len 12288 data: 11 9 12 23/30 Read block 225 len 4096 24/30 Read block 111 len 12288 25/30 Read block 370 len 36864 26/30 Read block 78 len 28672 27/30 Write block 360 len 36864 data: 14 17 12 11 10 13 14 12 12 28/30 Write block 566 len 40960 data: 10 12 11 12 13 12 14 15 14 16 29/30 Write block 222 len 4096 data: 10 30/30 Read block 300 len 4096 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Read 3 Sent Done Done false 1006 Acked Write 7 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Read 9 Sent Done Done false 1010 Acked Read 5 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 9 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 1 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 0 0 27 Done 3 30 30 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:40.203133904Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:40.203419304Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1683,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 236 236 236 F F F 1 020-039 135 136 136 243 244 244 F F F <--- 2 040-059 135 135 135 243 243 243 F F F 3 060-079 132 132 132 240 240 240 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 130 130 130 237 237 237 F F F 6 120-139 134 136 136 241 244 244 T F F <--- 7 140-159 134 136 136 241 244 244 F F F <--- 8 160-179 135 135 135 243 243 243 F F F 9 180-199 132 132 132 240 240 240 F F F 10 200-219 135 136 136 243 244 244 F F F <--- 11 220-239 122 122 122 222 222 222 F T T <--- 12 240-259 134 134 134 242 242 242 F F F 13 260-279 131 131 131 239 239 239 F F F 14 280-299 132 132 132 240 240 240 F F F 15 300-319 132 132 132 240 240 240 F F F 16 320-339 130 130 130 238 238 238 F F F 17 340-359 132 132 132 240 240 240 F F F 18 360-379 135 135 135 243 243 243 F T T <--- 19 380-399 131 131 131 239 239 239 F F F 20 400-419 130 136 136 237 244 244 F F F <--- 21 420-439 135 135 135 243 243 243 F F F 22 440-459 135 135 135 243 243 243 F F F 23 460-479 132 132 132 240 240 240 F F F 24 480-499 134 134 134 242 242 242 F F F 25 500-519 134 134 134 242 242 242 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 135 136 136 243 244 244 F T T <--- 28 560-579 135 135 135 243 243 243 F T T <--- 29 580-599 126 126 126 232 232 232 F F F Max gen: 136, Max flush: 244 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 236 236 236 F F F 1 020-039 136 136 136 244 244 244 F F F 2 040-059 135 135 135 243 243 243 F F F 3 060-079 132 132 132 240 240 240 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 130 130 130 237 237 237 F F F 6 120-139 136 136 136 244 244 244 F F F 7 140-159 136 136 136 244 244 244 F F F 8 160-179 135 135 135 243 243 243 F F F 9 180-199 132 132 132 240 240 240 F F F 10 200-219 136 136 136 244 244 244 F F F 11 220-239 137 137 137 245 245 245 F F F 12 240-259 134 134 134 242 242 242 F F F 13 260-279 131 131 131 239 239 239 F F F 14 280-299 132 132 132 240 240 240 F F F 15 300-319 132 132 132 240 240 240 F F F 16 320-339 130 130 130 238 238 238 F F F 17 340-359 132 132 132 240 240 240 F F F 18 360-379 137 137 137 245 245 245 F F F 19 380-399 131 131 131 239 239 239 F F F 20 400-419 136 136 136 244 244 244 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 135 135 135 243 243 243 F F F 23 460-479 132 132 132 240 240 240 F F F 24 480-499 134 134 134 242 242 242 F F F 25 500-519 134 134 134 242 242 242 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 137 137 137 245 245 245 F F F 28 560-579 137 137 137 245 245 245 F F F 29 580-599 126 126 126 232 232 232 F F F Max gen: 137, Max flush: 245 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.44296117Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.451304482Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.451588823Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.451618191Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.45162706Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.451639Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691} {"msg":"Crucible f8d78cbf-fd0a-4d70-af06-01371c1937f3 has session id: ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.451954308Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"Upstairs opts: Upstairs UUID: f8d78cbf-fd0a-4d70-af06-01371c1937f3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.451987996Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"Crucible stats registered with UUID: f8d78cbf-fd0a-4d70-af06-01371c1937f3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.451999355Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:44.452011674Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.452188802Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.452206391Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.4522168Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.452315004Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.452328873Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691} {"msg":"f8d78cbf-fd0a-4d70-af06-01371c1937f3 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.45236055Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.4523705Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"client":"0","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.452380379Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"client":"1","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.452390308Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"client":"2","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.452400208Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.452410527Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.452420766Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"ds_connection connected from Ok(127.0.0.1:37291)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.452457414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"io task","client":"0","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"ds_connection connected from Ok(127.0.0.1:48044)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.452473463Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"io task","client":"2","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"ds_connection connected from Ok(127.0.0.1:40809)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.452484122Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"io task","client":"1","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453168715Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"client":"1","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453211093Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"client":"0","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453222522Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"client":"2","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453509162Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453556159Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"[0]R flush_numbers[0..12]: [236, 244, 243, 240, 237, 237, 244, 244, 243, 240, 244, 245]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453608565Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"downstairs","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"[0]R generation[0..12]: [128, 136, 135, 132, 130, 130, 136, 136, 135, 132, 136, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453620165Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"downstairs","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453631594Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"downstairs","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"[1]R flush_numbers[0..12]: [236, 244, 243, 240, 237, 237, 244, 244, 243, 240, 244, 245]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453642523Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"downstairs","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"[1]R generation[0..12]: [128, 136, 135, 132, 130, 130, 136, 136, 135, 132, 136, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453653292Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"downstairs","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453664432Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"downstairs","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"[2]R flush_numbers[0..12]: [236, 244, 243, 240, 237, 237, 244, 244, 243, 240, 244, 245]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453675231Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"downstairs","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"[2]R generation[0..12]: [128, 136, 135, 132, 130, 130, 136, 136, 135, 132, 136, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.45368606Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"downstairs","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453697389Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"downstairs","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453707189Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"downstairs","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453717258Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"downstairs","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453726957Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"downstairs","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453736477Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"":"downstairs","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453747996Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453757095Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"client":"0","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453767175Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"client":"1","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453777084Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"client":"2","":"downstairs","session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453787123Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453796553Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"f8d78cbf-fd0a-4d70-af06-01371c1937f3 is now active with session: ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453806822Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453816341Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.453824331Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"session_id":"ac360ce8-fa0f-49bb-ad2c-21eeeedd3ffe"} {"msg":"Activated sub_volume f8d78cbf-fd0a-4d70-af06-01371c1937f3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.45383401Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.4538426Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.454166068Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"task":"crutest"} 01/30 Flush 02/30 Write block 369 len 8192 data: 13 12 03/30 Write block 75 len 20480 data: 9 9 10 10 8 04/30 Read block 456 len 28672 05/30 Write block 467 len 12288 data: 18 18 17 06/30 Read block 282 len 16384 07/30 Read block 206 len 8192 08/30 Read block 568 len 4096 09/30 Read block 514 len 20480 10/30 Write block 576 len 24576 data: 15 13 13 13 13 10 11/30 Read block 12 len 8192 12/30 Read block 174 len 28672 13/30 Read block 82 len 32768 14/30 Write block 569 len 36864 data: 13 14 13 15 16 15 17 16 14 15/30 Read block 424 len 12288 16/30 Read block 346 len 24576 17/30 Flush 18/30 Write block 275 len 36864 data: 6 7 6 7 7 8 8 8 6 19/30 Write block 258 len 40960 data: 13 13 11 10 9 10 9 8 6 4 20/30 Read block 380 len 4096 21/30 Write block 146 len 24576 data: 17 16 15 16 17 18 22/30 Read block 554 len 20480 23/30 Flush 24/30 Write block 35 len 8192 data: 12 13 25/30 Flush 26/30 Write block 500 len 28672 data: 11 11 11 11 13 15 11 27/30 Read block 370 len 40960 28/30 Write block 558 len 28672 data: 10 13 12 12 12 14 13 29/30 Write block 336 len 4096 data: 14 30/30 Read block 522 len 32768 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 2 Done Done Done false 1002 Acked Write 5 Done Done Sent false 1003 Acked Read 7 Done Done Sent false 1004 Acked Write 3 Done Done Sent false 1005 Acked Read 4 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 7 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 28 29 Done 29 28 1 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:44.463050423Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:44.463346433Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1691,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 236 236 236 F F F 1 020-039 138 138 136 249 249 244 F F F <--- 2 040-059 135 135 135 243 243 243 F F F 3 060-079 138 138 132 247 247 240 F F F <--- 4 080-099 130 130 130 237 237 237 F F F 5 100-119 130 130 130 237 237 237 F F F 6 120-139 136 136 136 244 244 244 F F F 7 140-159 138 138 136 248 248 244 F F F <--- 8 160-179 135 135 135 243 243 243 F F F 9 180-199 132 132 132 240 240 240 F F F 10 200-219 136 136 136 244 244 244 F F F 11 220-239 137 137 137 245 245 245 F F F 12 240-259 138 138 134 248 248 242 F F F <--- 13 260-279 138 138 131 248 248 239 F F F <--- 14 280-299 138 138 132 248 248 240 F F F <--- 15 300-319 132 132 132 240 240 240 F F F 16 320-339 130 130 130 238 238 238 T T F <--- 17 340-359 132 132 132 240 240 240 F F F 18 360-379 138 138 137 247 247 245 F F T <--- 19 380-399 131 131 131 239 239 239 F F F 20 400-419 136 136 136 244 244 244 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 135 135 135 243 243 243 F F F 23 460-479 138 138 132 247 247 240 F F F <--- 24 480-499 134 134 134 242 242 242 F F F 25 500-519 134 134 134 242 242 242 T T F <--- 26 520-539 128 128 128 236 236 236 F F F 27 540-559 137 137 137 245 245 245 T T F <--- 28 560-579 138 138 137 247 247 245 T T F <--- 29 580-599 138 138 126 247 247 232 F F F <--- Max gen: 138, Max flush: 249 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 236 236 236 F F F 1 020-039 138 138 138 249 249 249 F F F 2 040-059 135 135 135 243 243 243 F F F 3 060-079 138 138 138 247 247 247 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 130 130 130 237 237 237 F F F 6 120-139 136 136 136 244 244 244 F F F 7 140-159 138 138 138 248 248 248 F F F 8 160-179 135 135 135 243 243 243 F F F 9 180-199 132 132 132 240 240 240 F F F 10 200-219 136 136 136 244 244 244 F F F 11 220-239 137 137 137 245 245 245 F F F 12 240-259 138 138 138 248 248 248 F F F 13 260-279 138 138 138 248 248 248 F F F 14 280-299 138 138 138 248 248 248 F F F 15 300-319 132 132 132 240 240 240 F F F 16 320-339 139 139 139 250 250 250 F F F 17 340-359 132 132 132 240 240 240 F F F 18 360-379 138 138 138 247 247 247 F F F 19 380-399 131 131 131 239 239 239 F F F 20 400-419 136 136 136 244 244 244 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 135 135 135 243 243 243 F F F 23 460-479 138 138 138 247 247 247 F F F 24 480-499 134 134 134 242 242 242 F F F 25 500-519 139 139 139 250 250 250 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 139 139 139 250 250 250 F F F 28 560-579 139 139 139 250 250 250 F F F 29 580-599 138 138 138 247 247 247 F F F Max gen: 139, Max flush: 250 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.705155715Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.713430442Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.713736021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.713778598Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.713787777Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.713802066Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699} {"msg":"Crucible 87033f97-4cf0-4a72-ac40-6d479ba7792b has session id: 9b04644b-ad36-465d-8a44-0d89c7781f79","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714156252Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"Upstairs opts: Upstairs UUID: 87033f97-4cf0-4a72-ac40-6d479ba7792b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.71418622Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"Crucible stats registered with UUID: 87033f97-4cf0-4a72-ac40-6d479ba7792b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714197789Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:48.714207479Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714381057Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714398896Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714409505Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714543606Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714562295Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699} {"msg":"87033f97-4cf0-4a72-ac40-6d479ba7792b active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714586283Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714595932Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"client":"0","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714606042Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"client":"1","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714615981Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"client":"2","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.71462605Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714638469Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714648919Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"ds_connection connected from Ok(127.0.0.1:62441)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714730203Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"io task","client":"1","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"ds_connection connected from Ok(127.0.0.1:45401)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.714756871Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"io task","client":"2","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"ds_connection connected from Ok(127.0.0.1:41724)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.71476853Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"io task","client":"0","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.715558297Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"client":"1","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.715585025Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"client":"2","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.715606833Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"client":"0","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.715897414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.715917202Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"[0]R flush_numbers[0..12]: [236, 249, 243, 247, 237, 237, 244, 248, 243, 240, 244, 245]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.715982038Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"downstairs","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"[0]R generation[0..12]: [128, 138, 135, 138, 130, 130, 136, 138, 135, 132, 136, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.715993777Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"downstairs","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716005396Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"downstairs","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"[1]R flush_numbers[0..12]: [236, 249, 243, 247, 237, 237, 244, 248, 243, 240, 244, 245]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716016656Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"downstairs","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"[1]R generation[0..12]: [128, 138, 135, 138, 130, 130, 136, 138, 135, 132, 136, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716029765Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"downstairs","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716041104Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"downstairs","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"[2]R flush_numbers[0..12]: [236, 249, 243, 247, 237, 237, 244, 248, 243, 240, 244, 245]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716051973Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"downstairs","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"[2]R generation[0..12]: [128, 138, 135, 138, 130, 130, 136, 138, 135, 132, 136, 137]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716062752Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"downstairs","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716073962Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"downstairs","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716083641Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"downstairs","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.71609385Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"downstairs","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.71610379Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"downstairs","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716113559Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"":"downstairs","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716123688Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716132748Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"client":"0","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716142707Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"client":"1","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716152586Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"client":"2","":"downstairs","session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716162686Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716172255Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"87033f97-4cf0-4a72-ac40-6d479ba7792b is now active with session: 9b04644b-ad36-465d-8a44-0d89c7781f79","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716183884Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716192864Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"session_id":"9b04644b-ad36-465d-8a44-0d89c7781f79"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716202203Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699} {"msg":"Activated sub_volume 87033f97-4cf0-4a72-ac40-6d479ba7792b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716210462Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716218912Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.716554179Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"task":"crutest"} 01/30 Write block 390 len 20480 data: 10 11 10 9 13 02/30 Read block 540 len 36864 03/30 Write block 194 len 8192 data: 13 14 04/30 Read block 385 len 32768 05/30 Flush 06/30 Write block 211 len 16384 data: 15 13 12 10 07/30 Flush 08/30 Write block 589 len 40960 data: 7 9 12 10 10 10 9 8 8 7 09/30 Flush 10/30 Write block 494 len 28672 data: 9 8 8 8 7 10 12 11/30 Read block 22 len 20480 12/30 Write block 53 len 24576 data: 7 8 9 10 10 10 13/30 Write block 266 len 40960 data: 7 5 7 7 5 5 6 5 4 7 14/30 Read block 220 len 12288 15/30 Read block 239 len 40960 16/30 Write block 317 len 12288 data: 19 21 19 17/30 Flush 18/30 Write block 399 len 36864 data: 15 13 13 13 11 10 10 9 9 19/30 Read block 598 len 8192 20/30 Write block 286 len 20480 data: 4 7 6 5 4 21/30 Flush 22/30 Write block 478 len 32768 data: 11 9 8 10 12 12 12 12 23/30 Write block 232 len 40960 data: 11 9 8 10 10 7 7 9 11 13 24/30 Write block 550 len 36864 data: 13 11 11 12 10 13 10 9 11 25/30 Read block 272 len 28672 26/30 Read block 440 len 12288 27/30 Write block 100 len 16384 data: 9 9 8 8 28/30 Read block 231 len 36864 29/30 Read block 9 len 24576 30/30 Read block 374 len 36864 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 7 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Read 2 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 30 3 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:48.726315985Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:48.726607865Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1699,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 236 236 236 F F F 1 020-039 138 138 138 249 249 249 F F F 2 040-059 140 135 140 254 243 254 F F F <--- 3 060-079 138 138 138 247 247 247 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 130 130 130 237 237 237 T F T <--- 6 120-139 136 136 136 244 244 244 F F F 7 140-159 138 138 138 248 248 248 F F F 8 160-179 135 135 135 243 243 243 F F F 9 180-199 140 132 140 251 240 251 F T F <--- 10 200-219 140 136 140 252 244 252 F F F <--- 11 220-239 137 137 137 245 245 245 T F T <--- 12 240-259 138 138 138 248 248 248 T F T <--- 13 260-279 140 138 140 254 248 254 F F F <--- 14 280-299 140 138 140 255 248 255 F F F <--- 15 300-319 140 132 140 254 240 254 F F F <--- 16 320-339 139 139 139 250 250 250 F F F 17 340-359 132 132 132 240 240 240 F F F 18 360-379 138 138 138 247 247 247 F F F 19 380-399 140 131 140 255 239 255 F T F <--- 20 400-419 140 136 140 255 244 255 F F F <--- 21 420-439 135 135 135 243 243 243 F F F 22 440-459 135 135 135 243 243 243 F F F 23 460-479 138 138 138 247 247 247 T F T <--- 24 480-499 140 134 140 254 242 254 T F T <--- 25 500-519 140 139 140 254 250 254 F F F <--- 26 520-539 128 128 128 236 236 236 F F F 27 540-559 139 139 139 250 250 250 T F T <--- 28 560-579 139 139 139 250 250 250 F F F 29 580-599 140 138 140 253 247 253 F F F <--- Max gen: 140, Max flush: 255 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 236 236 236 F F F 1 020-039 138 138 138 249 249 249 F F F 2 040-059 140 140 140 254 254 254 F F F 3 060-079 138 138 138 247 247 247 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 141 141 141 256 256 256 F F F 6 120-139 136 136 136 244 244 244 F F F 7 140-159 138 138 138 248 248 248 F F F 8 160-179 135 135 135 243 243 243 F F F 9 180-199 140 140 140 251 251 251 F F F 10 200-219 140 140 140 252 252 252 F F F 11 220-239 141 141 141 256 256 256 F F F 12 240-259 141 141 141 256 256 256 F F F 13 260-279 140 140 140 254 254 254 F F F 14 280-299 140 140 140 255 255 255 F F F 15 300-319 140 140 140 254 254 254 F F F 16 320-339 139 139 139 250 250 250 F F F 17 340-359 132 132 132 240 240 240 F F F 18 360-379 138 138 138 247 247 247 F F F 19 380-399 140 140 140 255 255 255 F F F 20 400-419 140 140 140 255 255 255 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 135 135 135 243 243 243 F F F 23 460-479 141 141 141 256 256 256 F F F 24 480-499 141 141 141 256 256 256 F F F 25 500-519 140 140 140 254 254 254 F F F 26 520-539 128 128 128 236 236 236 F F F 27 540-559 141 141 141 256 256 256 F F F 28 560-579 139 139 139 250 250 250 F F F 29 580-599 140 140 140 253 253 253 F F F Max gen: 141, Max flush: 256 Begin loop 71 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.979500032Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.987674936Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.987952827Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.987978865Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.987987955Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.987999644Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707} {"msg":"Crucible 256b12aa-6564-48d4-8b21-383cc84109ac has session id: 03c26ced-d6c1-40c6-900c-e6504070f943","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988367769Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"Upstairs opts: Upstairs UUID: 256b12aa-6564-48d4-8b21-383cc84109ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988390267Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"Crucible stats registered with UUID: 256b12aa-6564-48d4-8b21-383cc84109ac","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988400427Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:52.988415026Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988622382Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988638831Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.98864922Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988783641Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988801949Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707} {"msg":"256b12aa-6564-48d4-8b21-383cc84109ac active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988840357Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988850046Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"client":"0","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988860065Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"client":"1","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988869895Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"client":"2","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988879834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988890103Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988900313Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"ds_connection connected from Ok(127.0.0.1:65046)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.988995986Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"io task","client":"2","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"ds_connection connected from Ok(127.0.0.1:65286)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.989070431Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"io task","client":"0","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"ds_connection connected from Ok(127.0.0.1:64011)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.989096199Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"io task","client":"1","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.989649362Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"client":"0","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.989692959Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"client":"1","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.989710158Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"client":"2","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.989976679Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.989987199Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"[0]R flush_numbers[0..12]: [236, 249, 254, 247, 237, 256, 244, 248, 243, 251, 252, 256]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990031856Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"downstairs","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"[0]R generation[0..12]: [128, 138, 140, 138, 130, 141, 136, 138, 135, 140, 140, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990043715Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"downstairs","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990055054Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"downstairs","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"[1]R flush_numbers[0..12]: [236, 249, 254, 247, 237, 256, 244, 248, 243, 251, 252, 256]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990066043Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"downstairs","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"[1]R generation[0..12]: [128, 138, 140, 138, 130, 141, 136, 138, 135, 140, 140, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990076813Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"downstairs","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990087982Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"downstairs","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"[2]R flush_numbers[0..12]: [236, 249, 254, 247, 237, 256, 244, 248, 243, 251, 252, 256]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990098941Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"downstairs","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"[2]R generation[0..12]: [128, 138, 140, 138, 130, 141, 136, 138, 135, 140, 140, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.99010974Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"downstairs","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.99012107Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"downstairs","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990133059Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"downstairs","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990143238Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"downstairs","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990152947Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"downstairs","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990162677Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"":"downstairs","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990173096Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990182325Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"client":"0","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990192495Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"client":"1","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990202564Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"client":"2","":"downstairs","session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990212903Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990222423Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"256b12aa-6564-48d4-8b21-383cc84109ac is now active with session: 03c26ced-d6c1-40c6-900c-e6504070f943","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990232582Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990241941Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990249791Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"session_id":"03c26ced-d6c1-40c6-900c-e6504070f943"} {"msg":"Activated sub_volume 256b12aa-6564-48d4-8b21-383cc84109ac","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.99025917Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.9902675Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.990587628Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"task":"crutest"} 01/30 Write block 114 len 8192 data: 9 8 02/30 Write block 520 len 40960 data: 13 11 11 10 12 14 13 13 12 12 03/30 Write block 561 len 4096 data: 13 04/30 Flush 05/30 Write block 306 len 28672 data: 15 16 13 14 14 17 17 06/30 Read block 363 len 32768 07/30 Flush 08/30 Write block 591 len 8192 data: 13 11 09/30 Write block 392 len 36864 data: 11 10 14 13 13 15 15 16 14 10/30 Read block 17 len 24576 11/30 Write block 180 len 12288 data: 5 6 8 12/30 Write block 10 len 24576 data: 10 10 10 11 8 6 13/30 Read block 528 len 8192 14/30 Write block 384 len 40960 data: 9 11 9 6 7 9 11 12 12 11 15/30 Read block 340 len 40960 16/30 Flush 17/30 Write block 528 len 28672 data: 13 13 12 12 14 14 14 18/30 Read block 370 len 40960 19/30 Read block 577 len 40960 20/30 Write block 43 len 32768 data: 10 11 11 11 10 10 9 10 21/30 Write block 30 len 32768 data: 12 14 11 15 13 13 14 10 22/30 Write block 273 len 28672 data: 6 5 8 8 7 8 8 23/30 Read block 452 len 20480 24/30 Write block 162 len 16384 data: 12 12 13 14 25/30 Flush 26/30 Write block 136 len 40960 data: 11 13 15 16 14 15 15 15 18 16 27/30 Read block 44 len 4096 28/30 Write block 520 len 24576 data: 14 12 12 11 13 15 29/30 Read block 454 len 16384 30/30 Read block 6 len 40960 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 2 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Read 10 Done Sent Done false 1018 Acked Read 10 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 16 1 17 Done 23 7 22 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1006)) Some(JobId(1024)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:52.999639222Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:52.999948061Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1707,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 128 142 259 236 259 F T F <--- 1 020-039 142 138 142 260 249 260 F F F <--- 2 040-059 142 140 142 260 254 260 F F F <--- 3 060-079 138 138 138 247 247 247 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 142 142 142 257 257 257 F F F 6 120-139 136 136 136 244 244 244 T F T <--- 7 140-159 138 138 138 248 248 248 T F T <--- 8 160-179 142 135 142 260 243 260 F F F <--- 9 180-199 142 140 142 259 251 259 F T F <--- 10 200-219 140 140 140 252 252 252 F F F 11 220-239 141 141 141 256 256 256 F F F 12 240-259 141 141 141 256 256 256 F F F 13 260-279 142 140 142 260 254 260 F F F <--- 14 280-299 140 140 140 255 255 255 F F F 15 300-319 142 142 142 258 258 258 F F F 16 320-339 139 139 139 250 250 250 F F F 17 340-359 132 132 132 240 240 240 F F F 18 360-379 138 138 138 247 247 247 F F F 19 380-399 142 140 142 259 255 259 F T F <--- 20 400-419 142 140 142 259 255 259 F T F <--- 21 420-439 135 135 135 243 243 243 F F F 22 440-459 135 135 135 243 243 243 F F F 23 460-479 141 141 141 256 256 256 F F F 24 480-499 141 141 141 256 256 256 F F F 25 500-519 140 140 140 254 254 254 F F F 26 520-539 142 142 142 260 257 260 T F T <--- 27 540-559 141 141 141 256 256 256 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 142 140 142 259 253 259 F T F <--- Max gen: 142, Max flush: 260 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 259 259 259 F F F 1 020-039 142 142 142 260 260 260 F F F 2 040-059 142 142 142 260 260 260 F F F 3 060-079 138 138 138 247 247 247 F F F 4 080-099 130 130 130 237 237 237 F F F 5 100-119 142 142 142 257 257 257 F F F 6 120-139 143 143 143 261 261 261 F F F 7 140-159 143 143 143 261 261 261 F F F 8 160-179 142 142 142 260 260 260 F F F 9 180-199 142 142 142 259 259 259 F F F 10 200-219 140 140 140 252 252 252 F F F 11 220-239 141 141 141 256 256 256 F F F 12 240-259 141 141 141 256 256 256 F F F 13 260-279 142 142 142 260 260 260 F F F 14 280-299 140 140 140 255 255 255 F F F 15 300-319 142 142 142 258 258 258 F F F 16 320-339 139 139 139 250 250 250 F F F 17 340-359 132 132 132 240 240 240 F F F 18 360-379 138 138 138 247 247 247 F F F 19 380-399 142 142 142 259 259 259 F F F 20 400-419 142 142 142 259 259 259 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 135 135 135 243 243 243 F F F 23 460-479 141 141 141 256 256 256 F F F 24 480-499 141 141 141 256 256 256 F F F 25 500-519 140 140 140 254 254 254 F F F 26 520-539 143 143 143 261 261 261 F F F 27 540-559 141 141 141 256 256 256 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 142 142 142 259 259 259 F F F Max gen: 143, Max flush: 261 Begin loop 72 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.247585916Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.255716553Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256006343Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256033311Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256042141Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.25605387Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715} {"msg":"Crucible 92eb1636-17a6-413e-9dd2-0ac53fc20192 has session id: b9617a02-7fcd-4661-94ec-47635f2852b9","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256354779Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"Upstairs opts: Upstairs UUID: 92eb1636-17a6-413e-9dd2-0ac53fc20192, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256395727Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"Crucible stats registered with UUID: 92eb1636-17a6-413e-9dd2-0ac53fc20192","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256406346Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:57.256418765Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256583164Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256614072Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256625341Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256772381Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256796859Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715} {"msg":"92eb1636-17a6-413e-9dd2-0ac53fc20192 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256816218Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256827617Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"client":"0","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256837817Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"client":"1","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256847716Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"client":"2","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256857655Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256868015Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256903472Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37875)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256946249Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"io task","client":"2","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:43209)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.256990476Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"io task","client":"0","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61716)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.257072221Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"io task","client":"1","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.257652671Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"client":"0","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.257683189Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"client":"1","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.257694948Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"client":"2","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.25795784Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.257995368Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"[0]R flush_numbers[0..12]: [259, 260, 260, 247, 237, 257, 261, 261, 260, 259, 252, 256]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258062663Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"downstairs","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"[0]R generation[0..12]: [142, 142, 142, 138, 130, 142, 143, 143, 142, 142, 140, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258077042Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"downstairs","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258088491Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"downstairs","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"[1]R flush_numbers[0..12]: [259, 260, 260, 247, 237, 257, 261, 261, 260, 259, 252, 256]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258099511Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"downstairs","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"[1]R generation[0..12]: [142, 142, 142, 138, 130, 142, 143, 143, 142, 142, 140, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.25811032Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"downstairs","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258121659Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"downstairs","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"[2]R flush_numbers[0..12]: [259, 260, 260, 247, 237, 257, 261, 261, 260, 259, 252, 256]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258132518Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"downstairs","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"[2]R generation[0..12]: [142, 142, 142, 138, 130, 142, 143, 143, 142, 142, 140, 141]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258143338Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"downstairs","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258154477Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"downstairs","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258164216Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"downstairs","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258174326Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"downstairs","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258183915Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"downstairs","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258193684Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"":"downstairs","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258203714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258212783Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"client":"0","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258224122Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"client":"1","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258234072Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"client":"2","":"downstairs","session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258244321Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.25825371Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"92eb1636-17a6-413e-9dd2-0ac53fc20192 is now active with session: b9617a02-7fcd-4661-94ec-47635f2852b9","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.2582638Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258273499Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258281498Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"session_id":"b9617a02-7fcd-4661-94ec-47635f2852b9"} {"msg":"Activated sub_volume 92eb1636-17a6-413e-9dd2-0ac53fc20192","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258290888Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258299477Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.258610026Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"task":"crutest"} 01/30 Write block 194 len 8192 data: 14 15 02/30 Read block 134 len 20480 03/30 Write block 469 len 40960 data: 18 17 15 12 11 8 9 8 8 12 04/30 Flush 05/30 Write block 200 len 16384 data: 13 12 9 9 06/30 Read block 225 len 20480 07/30 Write block 386 len 40960 data: 10 7 8 10 12 13 13 12 15 14 08/30 Write block 55 len 16384 data: 10 11 11 11 09/30 Write block 28 len 24576 data: 13 10 13 15 12 16 10/30 Read block 330 len 24576 11/30 Write block 237 len 28672 data: 8 8 10 12 14 15 14 12/30 Write block 480 len 8192 data: 9 11 13/30 Write block 291 len 8192 data: 5 7 14/30 Flush 15/30 Write block 25 len 8192 data: 11 11 16/30 Flush 17/30 Write block 176 len 8192 data: 8 6 18/30 Write block 456 len 20480 data: 9 11 14 12 12 19/30 Read block 128 len 24576 20/30 Write block 171 len 4096 data: 13 21/30 Read block 134 len 8192 22/30 Flush 23/30 Write block 549 len 32768 data: 15 14 12 12 13 11 14 11 24/30 Read block 190 len 32768 25/30 Read block 355 len 12288 26/30 Write block 96 len 12288 data: 10 12 14 27/30 Write block 305 len 8192 data: 12 16 28/30 Read block 61 len 28672 29/30 Read block 92 len 32768 30/30 Write block 35 len 32768 data: 14 15 11 11 12 12 11 11 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 4 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Write 10 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Read 6 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Write 2 Done Done Sent false 1012 Acked Write 2 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 2 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Write 1 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Write 3 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 21 23 Done 25 25 5 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:43:57.266644339Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:43:57.26692211Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1715,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 259 259 259 F F F 1 020-039 144 144 142 264 264 260 T T T <--- 2 040-059 144 144 142 263 263 260 T T T <--- 3 060-079 138 138 138 247 247 247 F F F 4 080-099 130 130 130 237 237 237 T T F <--- 5 100-119 142 142 142 257 257 257 F F F 6 120-139 143 143 143 261 261 261 F F F 7 140-159 143 143 143 261 261 261 F F F 8 160-179 144 144 142 265 265 260 F F F <--- 9 180-199 144 144 144 262 262 262 F F F 10 200-219 144 144 140 263 263 252 F F T <--- 11 220-239 144 144 141 263 263 256 F F F <--- 12 240-259 144 144 141 263 263 256 F F F <--- 13 260-279 142 142 142 260 260 260 F F F 14 280-299 144 144 140 263 263 255 F F F <--- 15 300-319 142 142 142 258 258 258 T T F <--- 16 320-339 139 139 139 250 250 250 F F F 17 340-359 132 132 132 240 240 240 F F F 18 360-379 138 138 138 247 247 247 F F F 19 380-399 144 144 142 263 263 259 F F T <--- 20 400-419 142 142 142 259 259 259 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 144 144 135 265 265 243 F F F <--- 23 460-479 144 144 144 265 265 262 F F F <--- 24 480-499 144 144 141 263 263 256 F F F <--- 25 500-519 140 140 140 254 254 254 F F F 26 520-539 143 143 143 261 261 261 F F F 27 540-559 141 141 141 256 256 256 T T F <--- 28 560-579 142 142 142 257 257 257 F F F 29 580-599 142 142 142 259 259 259 F F F Max gen: 144, Max flush: 265 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 259 259 259 F F F 1 020-039 145 145 145 266 266 266 F F F 2 040-059 145 145 145 266 266 266 F F F 3 060-079 138 138 138 247 247 247 F F F 4 080-099 145 145 145 266 266 266 F F F 5 100-119 142 142 142 257 257 257 F F F 6 120-139 143 143 143 261 261 261 F F F 7 140-159 143 143 143 261 261 261 F F F 8 160-179 144 144 144 265 265 265 F F F 9 180-199 144 144 144 262 262 262 F F F 10 200-219 144 144 144 263 263 263 F F F 11 220-239 144 144 144 263 263 263 F F F 12 240-259 144 144 144 263 263 263 F F F 13 260-279 142 142 142 260 260 260 F F F 14 280-299 144 144 144 263 263 263 F F F 15 300-319 145 145 145 266 266 266 F F F 16 320-339 139 139 139 250 250 250 F F F 17 340-359 132 132 132 240 240 240 F F F 18 360-379 138 138 138 247 247 247 F F F 19 380-399 144 144 144 263 263 263 F F F 20 400-419 142 142 142 259 259 259 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 144 144 144 265 265 265 F F F 23 460-479 144 144 144 265 265 265 F F F 24 480-499 144 144 144 263 263 263 F F F 25 500-519 140 140 140 254 254 254 F F F 26 520-539 143 143 143 261 261 261 F F F 27 540-559 145 145 145 266 266 266 F F F 28 560-579 142 142 142 257 257 257 F F F 29 580-599 142 142 142 259 259 259 F F F Max gen: 145, Max flush: 266 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.514119355Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:01.521217252Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.521277228Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.528187148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.528433372Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.52846026Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.528469409Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.528483598Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723} {"msg":"Crucible 57438f7f-394a-422e-ad42-5b4e2ac53308 has session id: 1625cbf1-13a8-4e38-9220-590088ae47a3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.528857353Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"Upstairs opts: Upstairs UUID: 57438f7f-394a-422e-ad42-5b4e2ac53308, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.528875501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"Crucible stats registered with UUID: 57438f7f-394a-422e-ad42-5b4e2ac53308","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.528886151Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:01.52889882Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.529065059Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.529081967Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.529092447Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.529236107Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.529261305Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723} {"msg":"57438f7f-394a-422e-ad42-5b4e2ac53308 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.529279724Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.529288973Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"client":"0","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.529299163Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"client":"1","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.529309192Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"client":"2","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.529319231Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.52933148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.52934191Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:35357)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.529537626Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"io task","client":"1","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:41341)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.529565844Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"io task","client":"0","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"ds_connection connected from Ok(127.0.0.1:49500)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.529577254Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"io task","client":"2","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530196032Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"client":"1","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530256767Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"client":"0","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530293235Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"client":"2","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530539508Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530578316Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"[0]R flush_numbers[0..12]: [259, 266, 266, 247, 266, 257, 261, 261, 265, 262, 263, 263]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530629952Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"downstairs","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"[0]R generation[0..12]: [142, 145, 145, 138, 145, 142, 143, 143, 144, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530641441Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"downstairs","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530652831Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"downstairs","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"[1]R flush_numbers[0..12]: [259, 266, 266, 247, 266, 257, 261, 261, 265, 262, 263, 263]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.53066377Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"downstairs","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"[1]R generation[0..12]: [142, 145, 145, 138, 145, 142, 143, 143, 144, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530676789Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"downstairs","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530688208Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"downstairs","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"[2]R flush_numbers[0..12]: [259, 266, 266, 247, 266, 257, 261, 261, 265, 262, 263, 263]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530698997Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"downstairs","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"[2]R generation[0..12]: [142, 145, 145, 138, 145, 142, 143, 143, 144, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530709707Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"downstairs","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530720886Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"downstairs","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530730595Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"downstairs","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530740595Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"downstairs","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530750254Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"downstairs","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530760053Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"":"downstairs","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530770473Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530779692Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"client":"0","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530789931Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"client":"1","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.53080021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"client":"2","":"downstairs","session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.53081051Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530819999Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"57438f7f-394a-422e-ad42-5b4e2ac53308 is now active with session: 1625cbf1-13a8-4e38-9220-590088ae47a3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530830238Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530840898Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"session_id":"1625cbf1-13a8-4e38-9220-590088ae47a3"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530850317Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723} {"msg":"Activated sub_volume 57438f7f-394a-422e-ad42-5b4e2ac53308","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530858736Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.530867396Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.531194764Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"task":"crutest"} 01/30 Flush 02/30 Write block 155 len 8192 data: 8 10 03/30 Read block 46 len 20480 04/30 Read block 149 len 8192 05/30 Write block 197 len 16384 data: 10 11 13 14 06/30 Read block 364 len 20480 07/30 Read block 342 len 16384 08/30 Read block 436 len 20480 09/30 Write block 202 len 4096 data: 10 10/30 Write block 384 len 20480 data: 10 12 11 8 9 11/30 Write block 390 len 8192 data: 13 14 12/30 Read block 85 len 4096 13/30 Write block 575 len 24576 data: 18 17 15 14 14 14 14/30 Read block 360 len 8192 15/30 Write block 146 len 8192 data: 18 17 16/30 Write block 368 len 32768 data: 13 14 13 11 14 13 14 14 17/30 Write block 377 len 16384 data: 13 16 14 13 18/30 Read block 579 len 8192 19/30 Read block 475 len 12288 20/30 Write block 526 len 4096 data: 14 21/30 Read block 378 len 12288 22/30 Read block 144 len 16384 23/30 Read block 57 len 24576 24/30 Write block 357 len 32768 data: 11 11 14 15 18 13 12 11 25/30 Write block 350 len 20480 data: 13 13 11 10 10 26/30 Read block 186 len 28672 27/30 Read block 194 len 40960 28/30 Write block 327 len 20480 data: 11 13 15 12 12 29/30 Write block 118 len 12288 data: 9 10 8 30/30 Read block 342 len 24576 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 2 Done Done Done false 1002 Acked Read 5 Sent Done Done false 1003 Acked Read 2 Sent Done Done false 1004 Acked Write 4 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Read 5 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Write 2 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Write 8 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Write 3 Sent Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 0 1 29 Done 1 29 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:01.539237996Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:01.539539766Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1723,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 259 259 259 F F F 1 020-039 145 145 145 266 266 266 F F F 2 040-059 145 145 145 266 266 266 F F F 3 060-079 138 138 138 247 247 247 F F F 4 080-099 145 145 145 266 266 266 F F F 5 100-119 142 142 142 257 257 257 F T T <--- 6 120-139 143 143 143 261 261 261 F T T <--- 7 140-159 143 143 143 261 261 261 T T T 8 160-179 144 144 144 265 265 265 F F F 9 180-199 144 144 144 262 262 262 F T T <--- 10 200-219 144 144 144 263 263 263 F T T <--- 11 220-239 144 144 144 263 263 263 F F F 12 240-259 144 144 144 263 263 263 F F F 13 260-279 142 142 142 260 260 260 F F F 14 280-299 144 144 144 263 263 263 F F F 15 300-319 145 145 145 266 266 266 F F F 16 320-339 139 139 139 250 250 250 F T T <--- 17 340-359 132 132 132 240 240 240 F T T <--- 18 360-379 138 138 138 247 247 247 F T T <--- 19 380-399 144 144 144 263 263 263 F T T <--- 20 400-419 142 142 142 259 259 259 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 144 144 144 265 265 265 F F F 23 460-479 144 144 144 265 265 265 F F F 24 480-499 144 144 144 263 263 263 F F F 25 500-519 140 140 140 254 254 254 F F F 26 520-539 143 143 143 261 261 261 F T T <--- 27 540-559 145 145 145 266 266 266 F F F 28 560-579 142 142 142 257 257 257 F T T <--- 29 580-599 142 142 142 259 259 259 F T T <--- Max gen: 145, Max flush: 266 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 73 Verifying data now Files /tmp/test_repair/test_repair_verify.data and /tmp/test_repair/test_repair_verify.data.last differ 148a149 > 18, 150d150 < 16, diff found after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 259 259 259 F F F 1 020-039 145 145 145 266 266 266 F F F 2 040-059 145 145 145 266 266 266 F F F 3 060-079 138 138 138 247 247 247 F F F 4 080-099 145 145 145 266 266 266 F F F 5 100-119 146 146 146 267 267 267 F F F 6 120-139 146 146 146 267 267 267 F F F 7 140-159 146 146 146 267 267 267 F F F 8 160-179 144 144 144 265 265 265 F F F 9 180-199 146 146 146 267 267 267 F F F 10 200-219 146 146 146 267 267 267 F F F 11 220-239 144 144 144 263 263 263 F F F 12 240-259 144 144 144 263 263 263 F F F 13 260-279 142 142 142 260 260 260 F F F 14 280-299 144 144 144 263 263 263 F F F 15 300-319 145 145 145 266 266 266 F F F 16 320-339 146 146 146 267 267 267 F F F 17 340-359 146 146 146 267 267 267 F F F 18 360-379 146 146 146 267 267 267 F F F 19 380-399 146 146 146 267 267 267 F F F 20 400-419 142 142 142 259 259 259 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 144 144 144 265 265 265 F F F 23 460-479 144 144 144 265 265 265 F F F 24 480-499 144 144 144 263 263 263 F F F 25 500-519 140 140 140 254 254 254 F F F 26 520-539 146 146 146 267 267 267 F F F 27 540-559 145 145 145 266 266 266 F F F 28 560-579 146 146 146 267 267 267 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 146, Max flush: 267 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.797427693Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.805611597Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.805896837Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.805924415Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.805933515Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.805945384Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734} {"msg":"Crucible 99dc38c9-8535-4969-8cc0-10491dab6cd1 has session id: f4014b82-c0ed-4935-93ed-dca5b340303f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806360416Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"Upstairs opts: Upstairs UUID: 99dc38c9-8535-4969-8cc0-10491dab6cd1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806390404Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"Crucible stats registered with UUID: 99dc38c9-8535-4969-8cc0-10491dab6cd1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806402223Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:05.806414172Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806571501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.80658879Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806599739Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806717671Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.80673582Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734} {"msg":"99dc38c9-8535-4969-8cc0-10491dab6cd1 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806770358Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806780137Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"client":"0","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806790236Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"client":"1","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806800336Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"client":"2","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806810395Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806835663Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41696)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806875011Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.80688597Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56417)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.806896589Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63382)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.807042529Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.807645688Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"client":"0","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.807666577Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"client":"2","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.807753941Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"client":"1","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.807997714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808074479Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"[0]R flush_numbers[0..12]: [259, 266, 266, 247, 266, 267, 267, 267, 265, 267, 267, 263]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808116476Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"downstairs","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"[0]R generation[0..12]: [142, 145, 145, 138, 145, 146, 146, 146, 144, 146, 146, 144]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808128195Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"downstairs","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808139615Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"downstairs","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"[1]R flush_numbers[0..12]: [259, 266, 266, 247, 266, 267, 267, 267, 265, 267, 267, 263]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808150524Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"downstairs","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"[1]R generation[0..12]: [142, 145, 145, 138, 145, 146, 146, 146, 144, 146, 146, 144]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808161393Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"downstairs","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808172722Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"downstairs","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"[2]R flush_numbers[0..12]: [259, 266, 266, 247, 266, 267, 267, 267, 265, 267, 267, 263]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808183652Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"downstairs","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"[2]R generation[0..12]: [142, 145, 145, 138, 145, 146, 146, 146, 144, 146, 146, 144]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808194811Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"downstairs","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.80820606Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"downstairs","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"Max found gen is 147","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808218399Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"downstairs","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"Generation requested: 148 >= found:147","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808228589Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"downstairs","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808238368Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"downstairs","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808248137Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"":"downstairs","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808258337Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808267566Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"client":"0","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808277895Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"client":"1","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808288155Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"client":"2","":"downstairs","session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808298504Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808308103Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"99dc38c9-8535-4969-8cc0-10491dab6cd1 is now active with session: f4014b82-c0ed-4935-93ed-dca5b340303f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808318532Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808328032Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808335991Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"session_id":"f4014b82-c0ed-4935-93ed-dca5b340303f"} {"msg":"Activated sub_volume 99dc38c9-8535-4969-8cc0-10491dab6cd1","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808345431Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.80835385Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.808680038Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"task":"crutest"} 01/30 Write block 301 len 20480 data: 11 11 11 13 13 02/30 Read block 534 len 32768 03/30 Read block 352 len 4096 04/30 Write block 47 len 20480 data: 11 11 10 11 11 05/30 Write block 253 len 36864 data: 8 10 10 11 12 14 14 12 11 06/30 Read block 426 len 36864 07/30 Read block 47 len 20480 08/30 Write block 547 len 40960 data: 14 15 16 15 13 13 14 12 15 12 09/30 Flush 10/30 Write block 134 len 4096 data: 10 11/30 Write block 75 len 24576 data: 10 10 11 11 9 9 12/30 Write block 5 len 40960 data: 11 11 14 13 13 11 11 11 12 9 13/30 Read block 59 len 12288 14/30 Read block 229 len 32768 15/30 Write block 491 len 16384 data: 9 9 11 10 16/30 Read block 120 len 4096 17/30 Read block 384 len 28672 18/30 Write block 70 len 40960 data: 5 5 7 8 9 11 11 12 12 10 19/30 Write block 330 len 16384 data: 13 13 14 15 20/30 Read block 191 len 36864 21/30 Read block 578 len 12288 22/30 Read block 130 len 4096 23/30 Read block 443 len 8192 24/30 Read block 62 len 36864 25/30 Read block 2 len 32768 26/30 Write block 349 len 20480 data: 12 14 14 12 11 27/30 Write block 459 len 16384 data: 13 13 11 9 28/30 Read block 129 len 16384 29/30 Read block 35 len 20480 30/30 Read block 315 len 32768 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 1 Done Sent Done false 1010 Acked Write 6 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Read 1 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Write 10 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Read 1 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 5 Done Sent Done false 1029 Acked Read 8 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 26 0 26 Done 30 4 30 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) None Some(JobId(1008)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:05.817895891Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:05.818186651Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1734,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 259 259 259 T F T <--- 1 020-039 145 145 145 266 266 266 F F F 2 040-059 148 145 148 268 266 268 F T F <--- 3 060-079 138 138 138 247 247 247 T F T <--- 4 080-099 145 145 145 266 266 266 T F T <--- 5 100-119 146 146 146 267 267 267 F F F 6 120-139 146 146 146 267 267 267 T F T <--- 7 140-159 146 146 146 267 267 267 F F F 8 160-179 144 144 144 265 265 265 F F F 9 180-199 146 146 146 267 267 267 F F F 10 200-219 146 146 146 267 267 267 F F F 11 220-239 144 144 144 263 263 263 F F F 12 240-259 148 144 148 268 263 268 F F F <--- 13 260-279 148 142 148 268 260 268 F F F <--- 14 280-299 144 144 144 263 263 263 F F F 15 300-319 148 145 148 268 266 268 F T F <--- 16 320-339 146 146 146 267 267 267 T F T <--- 17 340-359 146 146 146 267 267 267 T F T <--- 18 360-379 146 146 146 267 267 267 F F F 19 380-399 146 146 146 267 267 267 F F F 20 400-419 142 142 142 259 259 259 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 144 144 144 265 265 265 T F T <--- 23 460-479 144 144 144 265 265 265 T F T <--- 24 480-499 144 144 144 263 263 263 T F T <--- 25 500-519 140 140 140 254 254 254 F F F 26 520-539 146 146 146 267 267 267 F F F 27 540-559 148 145 148 268 266 268 F F F <--- 28 560-579 146 146 146 267 267 267 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 148, Max flush: 268 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 149 149 149 269 269 269 F F F 1 020-039 145 145 145 266 266 266 F F F 2 040-059 148 148 148 268 268 268 F F F 3 060-079 149 149 149 269 269 269 F F F 4 080-099 149 149 149 269 269 269 F F F 5 100-119 146 146 146 267 267 267 F F F 6 120-139 149 149 149 269 269 269 F F F 7 140-159 146 146 146 267 267 267 F F F 8 160-179 144 144 144 265 265 265 F F F 9 180-199 146 146 146 267 267 267 F F F 10 200-219 146 146 146 267 267 267 F F F 11 220-239 144 144 144 263 263 263 F F F 12 240-259 148 148 148 268 268 268 F F F 13 260-279 148 148 148 268 268 268 F F F 14 280-299 144 144 144 263 263 263 F F F 15 300-319 148 148 148 268 268 268 F F F 16 320-339 149 149 149 269 269 269 F F F 17 340-359 149 149 149 269 269 269 F F F 18 360-379 146 146 146 267 267 267 F F F 19 380-399 146 146 146 267 267 267 F F F 20 400-419 142 142 142 259 259 259 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 149 149 149 269 269 269 F F F 24 480-499 149 149 149 269 269 269 F F F 25 500-519 140 140 140 254 254 254 F F F 26 520-539 146 146 146 267 267 267 F F F 27 540-559 148 148 148 268 268 268 F F F 28 560-579 146 146 146 267 267 267 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 149, Max flush: 269 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.065822096Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.074024148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.074333887Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.074363605Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.074372694Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.074384643Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742} {"msg":"Crucible 0b870eb5-4eb4-46a7-9bbc-bd30fd2052e8 has session id: 03ae93ac-e5be-4740-877c-31e67845e74f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.074682943Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"Upstairs opts: Upstairs UUID: 0b870eb5-4eb4-46a7-9bbc-bd30fd2052e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.074699272Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"Crucible stats registered with UUID: 0b870eb5-4eb4-46a7-9bbc-bd30fd2052e8","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.074709581Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:10.074721801Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.074896479Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.074913108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.074923577Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075066127Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075081276Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742} {"msg":"0b870eb5-4eb4-46a7-9bbc-bd30fd2052e8 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075094905Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075104175Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"client":"0","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075114274Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"client":"1","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075124273Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"client":"2","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075134312Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075144712Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075155161Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53643)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075229846Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"ds_connection connected from Ok(127.0.0.1:65061)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075307051Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41610)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075345058Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075884621Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"client":"0","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075931888Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"client":"2","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.075956847Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"client":"1","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076232958Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076278255Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"[0]R flush_numbers[0..12]: [269, 266, 268, 269, 269, 267, 269, 267, 265, 267, 267, 263]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076306883Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"downstairs","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"[0]R generation[0..12]: [149, 145, 148, 149, 149, 146, 149, 146, 144, 146, 146, 144]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076318472Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"downstairs","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076329981Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"downstairs","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"[1]R flush_numbers[0..12]: [269, 266, 268, 269, 269, 267, 269, 267, 265, 267, 267, 263]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076341Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"downstairs","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"[1]R generation[0..12]: [149, 145, 148, 149, 149, 146, 149, 146, 144, 146, 146, 144]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.07635219Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"downstairs","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076363899Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"downstairs","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"[2]R flush_numbers[0..12]: [269, 266, 268, 269, 269, 267, 269, 267, 265, 267, 267, 263]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076374868Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"downstairs","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"[2]R generation[0..12]: [149, 145, 148, 149, 149, 146, 149, 146, 144, 146, 146, 144]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076385727Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"downstairs","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076396887Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"downstairs","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076406686Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"downstairs","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076416865Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"downstairs","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076426625Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"downstairs","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076436434Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"":"downstairs","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076446653Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076457032Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"client":"0","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076467052Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"client":"1","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076477071Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"client":"2","":"downstairs","session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.07648716Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.07649683Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"0b870eb5-4eb4-46a7-9bbc-bd30fd2052e8 is now active with session: 03ae93ac-e5be-4740-877c-31e67845e74f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076507089Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076516568Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076524548Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"session_id":"03ae93ac-e5be-4740-877c-31e67845e74f"} {"msg":"Activated sub_volume 0b870eb5-4eb4-46a7-9bbc-bd30fd2052e8","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076534147Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076542637Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.076846106Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"task":"crutest"} 01/30 Write block 338 len 40960 data: 12 13 14 13 14 13 14 12 8 7 02/30 Read block 523 len 16384 03/30 Write block 240 len 20480 data: 13 15 16 15 13 04/30 Read block 232 len 4096 05/30 Read block 209 len 24576 06/30 Read block 134 len 16384 07/30 Write block 263 len 4096 data: 11 08/30 Read block 395 len 16384 09/30 Read block 583 len 16384 10/30 Write block 125 len 32768 data: 11 13 9 10 10 9 11 13 11/30 Read block 346 len 20480 12/30 Read block 521 len 40960 13/30 Write block 45 len 12288 data: 12 12 12 14/30 Write block 15 len 16384 data: 7 5 6 9 15/30 Read block 455 len 40960 16/30 Flush 17/30 Write block 354 len 8192 data: 11 12 18/30 Write block 282 len 32768 data: 9 7 6 5 5 8 7 6 19/30 Write block 505 len 32768 data: 16 12 14 12 14 14 12 7 20/30 Write block 491 len 36864 data: 10 10 12 11 9 9 9 8 11 21/30 Read block 573 len 28672 22/30 Write block 25 len 8192 data: 12 12 23/30 Read block 55 len 28672 24/30 Read block 17 len 28672 25/30 Write block 552 len 24576 data: 14 15 13 16 13 10 26/30 Read block 99 len 8192 27/30 Read block 113 len 12288 28/30 Read block 222 len 40960 29/30 Read block 336 len 40960 30/30 Write block 156 len 8192 data: 11 7 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 5 Done Done Sent false 1003 Acked Read 1 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Read 4 Done Done Sent false 1006 Acked Write 1 Done Done Sent false 1007 Acked Read 4 Done Done Sent false 1008 Acked Read 4 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Write 4 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 2 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 7 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 6 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:10.085524576Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:10.085829195Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1742,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 149 270 270 269 F F F <--- 1 020-039 145 145 145 266 266 266 T T F <--- 2 040-059 150 150 148 270 270 268 F F F <--- 3 060-079 149 149 149 269 269 269 F F F 4 080-099 149 149 149 269 269 269 F F F 5 100-119 146 146 146 267 267 267 F F F 6 120-139 150 150 149 270 270 269 F F F <--- 7 140-159 146 146 146 267 267 267 T T F <--- 8 160-179 144 144 144 265 265 265 F F F 9 180-199 146 146 146 267 267 267 F F F 10 200-219 146 146 146 267 267 267 F F F 11 220-239 144 144 144 263 263 263 F F F 12 240-259 150 150 148 270 270 268 F F F <--- 13 260-279 150 150 148 270 270 268 F F F <--- 14 280-299 144 144 144 263 263 263 T T F <--- 15 300-319 148 148 148 268 268 268 F F F 16 320-339 150 150 149 270 270 269 F F T <--- 17 340-359 150 150 149 270 270 269 T T T <--- 18 360-379 146 146 146 267 267 267 F F F 19 380-399 146 146 146 267 267 267 F F F 20 400-419 142 142 142 259 259 259 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 149 149 149 269 269 269 F F F 24 480-499 149 149 149 269 269 269 T T F <--- 25 500-519 140 140 140 254 254 254 T T F <--- 26 520-539 146 146 146 267 267 267 F F F 27 540-559 148 148 148 268 268 268 T T F <--- 28 560-579 146 146 146 267 267 267 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 150, Max flush: 270 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 270 270 270 F F F 1 020-039 151 151 151 271 271 271 F F F 2 040-059 150 150 150 270 270 270 F F F 3 060-079 149 149 149 269 269 269 F F F 4 080-099 149 149 149 269 269 269 F F F 5 100-119 146 146 146 267 267 267 F F F 6 120-139 150 150 150 270 270 270 F F F 7 140-159 151 151 151 271 271 271 F F F 8 160-179 144 144 144 265 265 265 F F F 9 180-199 146 146 146 267 267 267 F F F 10 200-219 146 146 146 267 267 267 F F F 11 220-239 144 144 144 263 263 263 F F F 12 240-259 150 150 150 270 270 270 F F F 13 260-279 150 150 150 270 270 270 F F F 14 280-299 151 151 151 271 271 271 F F F 15 300-319 148 148 148 268 268 268 F F F 16 320-339 150 150 150 270 270 270 F F F 17 340-359 151 151 151 271 271 271 F F F 18 360-379 146 146 146 267 267 267 F F F 19 380-399 146 146 146 267 267 267 F F F 20 400-419 142 142 142 259 259 259 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 149 149 149 269 269 269 F F F 24 480-499 151 151 151 271 271 271 F F F 25 500-519 151 151 151 271 271 271 F F F 26 520-539 146 146 146 267 267 267 F F F 27 540-559 151 151 151 271 271 271 F F F 28 560-579 146 146 146 267 267 267 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 151, Max flush: 271 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.328093565Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.336367043Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.336652133Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.336679061Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.336688201Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.3367026Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750} {"msg":"Crucible ddc878a4-f931-4cdf-8f51-430fd7a8dfad has session id: 2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337154119Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"Upstairs opts: Upstairs UUID: ddc878a4-f931-4cdf-8f51-430fd7a8dfad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337183437Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"Crucible stats registered with UUID: ddc878a4-f931-4cdf-8f51-430fd7a8dfad","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337194646Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:14.337204286Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337370904Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337388383Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337398812Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337522684Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337548282Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750} {"msg":"ddc878a4-f931-4cdf-8f51-430fd7a8dfad active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337567251Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.33757645Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"client":"0","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.33758684Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"client":"1","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337596819Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"client":"2","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337606968Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337619177Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337629397Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"ds_connection connected from Ok(127.0.0.1:38301)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337666994Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"ds_connection connected from Ok(127.0.0.1:50682)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337678313Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"ds_connection connected from Ok(127.0.0.1:37242)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.337688893Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338403044Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"client":"1","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338453141Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"client":"2","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338475259Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"client":"0","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338712873Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.33875495Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"[0]R flush_numbers[0..12]: [270, 271, 270, 269, 269, 267, 270, 271, 265, 267, 267, 263]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338814996Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"downstairs","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"[0]R generation[0..12]: [150, 151, 150, 149, 149, 146, 150, 151, 144, 146, 146, 144]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338826985Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"downstairs","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338838364Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"downstairs","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"[1]R flush_numbers[0..12]: [270, 271, 270, 269, 269, 267, 270, 271, 265, 267, 267, 263]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338849344Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"downstairs","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"[1]R generation[0..12]: [150, 151, 150, 149, 149, 146, 150, 151, 144, 146, 146, 144]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338862013Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"downstairs","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338873342Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"downstairs","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"[2]R flush_numbers[0..12]: [270, 271, 270, 269, 269, 267, 270, 271, 265, 267, 267, 263]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338884391Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"downstairs","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"[2]R generation[0..12]: [150, 151, 150, 149, 149, 146, 150, 151, 144, 146, 146, 144]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.33889526Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"downstairs","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.3389066Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"downstairs","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338916619Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"downstairs","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338926688Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"downstairs","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338936388Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"downstairs","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338946287Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"":"downstairs","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338956406Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338965716Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"client":"0","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338975795Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"client":"1","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338985884Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"client":"2","":"downstairs","session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.338995944Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.339019862Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"ddc878a4-f931-4cdf-8f51-430fd7a8dfad is now active with session: 2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.339030181Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.339041111Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.33904897Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"session_id":"2db08bb1-3cdb-4ce0-8ea4-9fe6072a6255"} {"msg":"Activated sub_volume ddc878a4-f931-4cdf-8f51-430fd7a8dfad","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.339058469Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.339067029Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.339386607Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"task":"crutest"} 01/30 Write block 186 len 40960 data: 9 8 12 11 11 11 13 13 15 16 02/30 Read block 593 len 4096 03/30 Write block 212 len 4096 data: 14 04/30 Read block 35 len 24576 05/30 Write block 270 len 36864 data: 6 6 7 7 6 9 9 8 9 06/30 Read block 588 len 32768 07/30 Write block 167 len 28672 data: 16 13 11 13 14 11 14 08/30 Read block 360 len 40960 09/30 Read block 7 len 4096 10/30 Read block 373 len 36864 11/30 Write block 189 len 32768 data: 12 12 12 14 14 16 17 11 12/30 Read block 507 len 12288 13/30 Read block 219 len 28672 14/30 Flush 15/30 Write block 28 len 20480 data: 14 11 14 16 13 16/30 Write block 5 len 28672 data: 12 12 15 14 14 12 12 17/30 Read block 565 len 12288 18/30 Write block 494 len 12288 data: 12 10 10 19/30 Read block 292 len 8192 20/30 Read block 225 len 36864 21/30 Read block 379 len 24576 22/30 Read block 288 len 40960 23/30 Read block 429 len 8192 24/30 Read block 116 len 12288 25/30 Read block 83 len 4096 26/30 Write block 215 len 28672 data: 13 14 12 13 14 13 13 27/30 Read block 2 len 4096 28/30 Write block 10 len 32768 data: 13 13 12 13 10 8 6 7 29/30 Read block 101 len 24576 30/30 Read block 186 len 20480 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 1 Done Done Sent false 1002 Acked Write 1 Done Done Sent false 1003 Acked Read 6 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Read 8 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Read 2 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Write 7 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 29 30 Done 29 30 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:14.348688534Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:14.348989924Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1750,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 270 270 270 T T F <--- 1 020-039 151 151 151 271 271 271 T T F <--- 2 040-059 150 150 150 270 270 270 F F F 3 060-079 149 149 149 269 269 269 F F F 4 080-099 149 149 149 269 269 269 F F F 5 100-119 146 146 146 267 267 267 F F F 6 120-139 150 150 150 270 270 270 F F F 7 140-159 151 151 151 271 271 271 F F F 8 160-179 152 152 144 272 272 265 F F F <--- 9 180-199 152 152 146 272 272 267 F F T <--- 10 200-219 152 152 146 272 272 267 T T F <--- 11 220-239 144 144 144 263 263 263 T T F <--- 12 240-259 150 150 150 270 270 270 F F F 13 260-279 152 152 150 272 272 270 F F F <--- 14 280-299 151 151 151 271 271 271 F F F 15 300-319 148 148 148 268 268 268 F F F 16 320-339 150 150 150 270 270 270 F F F 17 340-359 151 151 151 271 271 271 F F F 18 360-379 146 146 146 267 267 267 F F F 19 380-399 146 146 146 267 267 267 F F F 20 400-419 142 142 142 259 259 259 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 149 149 149 269 269 269 F F F 24 480-499 151 151 151 271 271 271 T T F <--- 25 500-519 151 151 151 271 271 271 F F F 26 520-539 146 146 146 267 267 267 F F F 27 540-559 151 151 151 271 271 271 F F F 28 560-579 146 146 146 267 267 267 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 152, Max flush: 272 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 273 273 273 F F F 1 020-039 153 153 153 273 273 273 F F F 2 040-059 150 150 150 270 270 270 F F F 3 060-079 149 149 149 269 269 269 F F F 4 080-099 149 149 149 269 269 269 F F F 5 100-119 146 146 146 267 267 267 F F F 6 120-139 150 150 150 270 270 270 F F F 7 140-159 151 151 151 271 271 271 F F F 8 160-179 152 152 152 272 272 272 F F F 9 180-199 152 152 152 272 272 272 F F F 10 200-219 153 153 153 273 273 273 F F F 11 220-239 153 153 153 273 273 273 F F F 12 240-259 150 150 150 270 270 270 F F F 13 260-279 152 152 152 272 272 272 F F F 14 280-299 151 151 151 271 271 271 F F F 15 300-319 148 148 148 268 268 268 F F F 16 320-339 150 150 150 270 270 270 F F F 17 340-359 151 151 151 271 271 271 F F F 18 360-379 146 146 146 267 267 267 F F F 19 380-399 146 146 146 267 267 267 F F F 20 400-419 142 142 142 259 259 259 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 149 149 149 269 269 269 F F F 24 480-499 153 153 153 273 273 273 F F F 25 500-519 151 151 151 271 271 271 F F F 26 520-539 146 146 146 267 267 267 F F F 27 540-559 151 151 151 271 271 271 F F F 28 560-579 146 146 146 267 267 267 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 153, Max flush: 273 Begin loop 77 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.58206116Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:18.589163536Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.589224922Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.596143762Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.596390165Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.596416003Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.596424852Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.596436852Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758} {"msg":"Crucible af6d58f1-a144-4d0e-b9c5-93f404fb4725 has session id: 780e9460-e2e2-47df-b3e9-7187e1be443a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.596783038Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"Upstairs opts: Upstairs UUID: af6d58f1-a144-4d0e-b9c5-93f404fb4725, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.596812746Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"Crucible stats registered with UUID: af6d58f1-a144-4d0e-b9c5-93f404fb4725","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.596826195Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:18.596836034Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.596999843Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597031011Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.59704229Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597137444Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597151453Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758} {"msg":"af6d58f1-a144-4d0e-b9c5-93f404fb4725 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597164522Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597173741Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"client":"0","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597183831Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"client":"1","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.59719375Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"client":"2","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597203929Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597214759Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597225018Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56458)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597269295Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44628)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597287444Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56182)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597300483Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597956148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"client":"2","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.597995756Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"client":"1","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598041782Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"client":"0","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598313894Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598344372Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"[0]R flush_numbers[0..12]: [273, 273, 270, 269, 269, 267, 270, 271, 272, 272, 273, 273]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598418157Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"downstairs","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"[0]R generation[0..12]: [153, 153, 150, 149, 149, 146, 150, 151, 152, 152, 153, 153]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598430106Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"downstairs","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598446065Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"downstairs","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"[1]R flush_numbers[0..12]: [273, 273, 270, 269, 269, 267, 270, 271, 272, 272, 273, 273]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598457424Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"downstairs","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"[1]R generation[0..12]: [153, 153, 150, 149, 149, 146, 150, 151, 152, 152, 153, 153]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598468533Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"downstairs","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598479923Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"downstairs","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"[2]R flush_numbers[0..12]: [273, 273, 270, 269, 269, 267, 270, 271, 272, 272, 273, 273]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598490992Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"downstairs","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"[2]R generation[0..12]: [153, 153, 150, 149, 149, 146, 150, 151, 152, 152, 153, 153]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598504531Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"downstairs","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.59851575Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"downstairs","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598525799Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"downstairs","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598536009Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"downstairs","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598545878Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"downstairs","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598555607Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"":"downstairs","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598565747Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598574936Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"client":"0","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598584945Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"client":"1","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598594995Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"client":"2","":"downstairs","session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598605254Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598614873Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"af6d58f1-a144-4d0e-b9c5-93f404fb4725 is now active with session: 780e9460-e2e2-47df-b3e9-7187e1be443a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598625333Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598634772Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598642802Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"session_id":"780e9460-e2e2-47df-b3e9-7187e1be443a"} {"msg":"Activated sub_volume af6d58f1-a144-4d0e-b9c5-93f404fb4725","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598652531Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.59866112Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.598921233Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"task":"crutest"} 01/30 Write block 291 len 40960 data: 6 8 10 11 11 12 14 16 14 13 02/30 Write block 155 len 12288 data: 9 12 8 03/30 Write block 2 len 12288 data: 7 7 9 04/30 Write block 558 len 32768 data: 12 14 13 14 13 15 14 13 05/30 Read block 145 len 20480 06/30 Write block 305 len 16384 data: 14 17 17 14 07/30 Write block 311 len 40960 data: 18 18 16 18 16 19 20 22 20 16 08/30 Read block 179 len 28672 09/30 Write block 68 len 8192 data: 5 7 10/30 Read block 250 len 16384 11/30 Read block 313 len 28672 12/30 Write block 355 len 12288 data: 13 12 12 13/30 Read block 387 len 28672 14/30 Read block 401 len 28672 15/30 Flush 16/30 Write block 105 len 40960 data: 7 7 11 13 12 12 11 8 8 10 17/30 Read block 502 len 4096 18/30 Write block 479 len 4096 data: 10 19/30 Read block 148 len 12288 20/30 Read block 245 len 28672 21/30 Write block 208 len 16384 data: 16 13 14 16 22/30 Write block 512 len 40960 data: 8 10 10 9 9 7 9 11 15 13 23/30 Read block 324 len 16384 24/30 Write block 241 len 28672 data: 16 17 16 14 12 15 11 25/30 Flush 26/30 Write block 35 len 24576 data: 15 16 12 12 13 13 27/30 Write block 347 len 32768 data: 8 6 13 15 15 13 12 12 28/30 Read block 383 len 40960 29/30 Write block 393 len 40960 data: 13 16 15 14 16 16 17 15 14 14 30/30 Write block 548 len 28672 data: 16 17 16 14 15 16 14 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Write 3 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Write 4 Sent Done Done false 1006 Acked Write 10 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Read 7 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Read 7 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Write 7 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 2 2 32 Done 2 28 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:18.607787579Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:18.608173023Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1758,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 154 154 273 274 274 F F F <--- 1 020-039 153 153 153 273 273 273 F T T <--- 2 040-059 150 150 150 270 270 270 F T T <--- 3 060-079 149 154 154 269 274 274 F F F <--- 4 080-099 149 149 149 269 269 269 F F F 5 100-119 146 154 154 267 275 275 F F F <--- 6 120-139 150 150 150 270 270 270 F F F 7 140-159 151 154 154 271 274 274 T F F <--- 8 160-179 152 152 152 272 272 272 F F F 9 180-199 152 152 152 272 272 272 F F F 10 200-219 153 154 154 273 275 275 F F F <--- 11 220-239 153 153 153 273 273 273 F F F 12 240-259 150 154 154 270 275 275 F F F <--- 13 260-279 152 152 152 272 272 272 F F F 14 280-299 151 154 154 271 274 274 T F F <--- 15 300-319 148 154 154 268 274 274 T F F <--- 16 320-339 150 154 154 270 274 274 F F F <--- 17 340-359 151 154 154 271 274 274 F T T <--- 18 360-379 146 146 146 267 267 267 F F F 19 380-399 146 146 146 267 267 267 F T T <--- 20 400-419 142 142 142 259 259 259 F T T <--- 21 420-439 135 135 135 243 243 243 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 149 154 154 269 275 275 F F F <--- 24 480-499 153 153 153 273 273 273 F F F 25 500-519 151 154 154 271 275 275 F F F <--- 26 520-539 146 154 154 267 275 275 F F F <--- 27 540-559 151 154 154 271 274 274 F T T <--- 28 560-579 146 154 154 267 274 274 F F F <--- 29 580-599 146 146 146 267 267 267 F F F Max gen: 154, Max flush: 275 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 274 274 274 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 155 155 155 276 276 276 F F F 3 060-079 154 154 154 274 274 274 F F F 4 080-099 149 149 149 269 269 269 F F F 5 100-119 154 154 154 275 275 275 F F F 6 120-139 150 150 150 270 270 270 F F F 7 140-159 154 154 154 274 274 274 F F F 8 160-179 152 152 152 272 272 272 F F F 9 180-199 152 152 152 272 272 272 F F F 10 200-219 154 154 154 275 275 275 F F F 11 220-239 153 153 153 273 273 273 F F F 12 240-259 154 154 154 275 275 275 F F F 13 260-279 152 152 152 272 272 272 F F F 14 280-299 154 154 154 274 274 274 F F F 15 300-319 154 154 154 274 274 274 F F F 16 320-339 154 154 154 274 274 274 F F F 17 340-359 155 155 155 276 276 276 F F F 18 360-379 146 146 146 267 267 267 F F F 19 380-399 155 155 155 276 276 276 F F F 20 400-419 155 155 155 276 276 276 F F F 21 420-439 135 135 135 243 243 243 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 154 154 154 275 275 275 F F F 24 480-499 153 153 153 273 273 273 F F F 25 500-519 154 154 154 275 275 275 F F F 26 520-539 154 154 154 275 275 275 F F F 27 540-559 155 155 155 276 276 276 F F F 28 560-579 154 154 154 274 274 274 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 155, Max flush: 276 Begin loop 78 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.873977552Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:22.881098347Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.881157013Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.888067313Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.888305787Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.888330735Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.888339835Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.888351654Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766} {"msg":"Crucible 661081ba-4708-4cd1-a208-7f5b3bbdeb7d has session id: d268f289-97c3-4ed0-bee1-6a35e883c54c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.88869965Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"Upstairs opts: Upstairs UUID: 661081ba-4708-4cd1-a208-7f5b3bbdeb7d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.888721769Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"Crucible stats registered with UUID: 661081ba-4708-4cd1-a208-7f5b3bbdeb7d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.888732098Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:22.888745517Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.888898537Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.888915436Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.888925785Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889057396Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889082834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766} {"msg":"661081ba-4708-4cd1-a208-7f5b3bbdeb7d active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889096063Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889105253Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"client":"0","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889115232Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"client":"1","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889125261Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"client":"2","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889135241Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.88914571Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889156129Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33978)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889205386Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57280)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889229104Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62095)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889240713Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889909708Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"client":"2","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889950755Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"client":"1","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.889983403Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"client":"0","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890301251Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890311211Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"[0]R flush_numbers[0..12]: [274, 276, 276, 274, 269, 275, 270, 274, 272, 272, 275, 273]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890353448Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"downstairs","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"[0]R generation[0..12]: [154, 155, 155, 154, 149, 154, 150, 154, 152, 152, 154, 153]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890365067Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"downstairs","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890376486Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"downstairs","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"[1]R flush_numbers[0..12]: [274, 276, 276, 274, 269, 275, 270, 274, 272, 272, 275, 273]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890387435Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"downstairs","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"[1]R generation[0..12]: [154, 155, 155, 154, 149, 154, 150, 154, 152, 152, 154, 153]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890398215Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"downstairs","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890409404Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"downstairs","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"[2]R flush_numbers[0..12]: [274, 276, 276, 274, 269, 275, 270, 274, 272, 272, 275, 273]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890420173Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"downstairs","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"[2]R generation[0..12]: [154, 155, 155, 154, 149, 154, 150, 154, 152, 152, 154, 153]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890431272Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"downstairs","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890442392Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"downstairs","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890451971Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"downstairs","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.89046342Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"downstairs","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.89047357Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"downstairs","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890483239Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"":"downstairs","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890493338Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890502468Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"client":"0","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890512327Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"client":"1","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890522246Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"client":"2","":"downstairs","session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890532256Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890541575Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"661081ba-4708-4cd1-a208-7f5b3bbdeb7d is now active with session: d268f289-97c3-4ed0-bee1-6a35e883c54c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890551874Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890561184Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890569093Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"session_id":"d268f289-97c3-4ed0-bee1-6a35e883c54c"} {"msg":"Activated sub_volume 661081ba-4708-4cd1-a208-7f5b3bbdeb7d","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890578652Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890587112Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.890880112Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"task":"crutest"} 01/30 Flush 02/30 Write block 124 len 32768 data: 8 12 14 10 11 11 10 12 03/30 Write block 495 len 24576 data: 11 11 10 9 12 13 04/30 Read block 7 len 28672 05/30 Write block 306 len 12288 data: 18 18 15 06/30 Write block 51 len 12288 data: 12 9 8 07/30 Write block 551 len 24576 data: 15 16 17 15 17 14 08/30 Write block 355 len 12288 data: 14 13 13 09/30 Write block 198 len 28672 data: 12 14 15 13 11 10 14 10/30 Write block 201 len 40960 data: 14 12 11 15 15 14 12 17 14 15 11/30 Read block 214 len 24576 12/30 Read block 553 len 8192 13/30 Read block 428 len 8192 14/30 Read block 11 len 16384 15/30 Read block 251 len 36864 16/30 Read block 157 len 4096 17/30 Write block 122 len 8192 data: 7 6 18/30 Read block 43 len 8192 19/30 Write block 269 len 8192 data: 8 7 20/30 Write block 416 len 20480 data: 14 12 11 11 12 21/30 Write block 402 len 8192 data: 15 12 22/30 Write block 85 len 24576 data: 5 6 6 7 8 9 23/30 Read block 451 len 12288 24/30 Read block 554 len 24576 25/30 Write block 265 len 12288 data: 9 8 6 26/30 Read block 357 len 16384 27/30 Read block 20 len 20480 28/30 Read block 289 len 40960 29/30 Read block 353 len 12288 30/30 Write block 396 len 8192 data: 15 17 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 8 Done Done Done false 1002 Acked Write 6 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Write 3 Sent Done Done false 1005 Acked Write 3 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Write 3 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Read 6 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Read 9 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Read 2 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Write 3 Sent Done Done false 1025 Acked Read 4 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 3 Sent Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 2 31 Done 1 28 27 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1000)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:22.897789982Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:22.898093841Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1766,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 274 274 274 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 155 155 155 276 276 276 F T T <--- 3 060-079 154 154 154 274 274 274 F F F 4 080-099 149 149 149 269 269 269 F T T <--- 5 100-119 154 154 154 275 275 275 F F F 6 120-139 150 150 150 270 270 270 T T T 7 140-159 154 154 154 274 274 274 F F F 8 160-179 152 152 152 272 272 272 F F F 9 180-199 152 152 152 272 272 272 F T T <--- 10 200-219 154 154 154 275 275 275 F T T <--- 11 220-239 153 153 153 273 273 273 F F F 12 240-259 154 154 154 275 275 275 F F F 13 260-279 152 152 152 272 272 272 F T T <--- 14 280-299 154 154 154 274 274 274 F F F 15 300-319 154 154 154 274 274 274 F T T <--- 16 320-339 154 154 154 274 274 274 F F F 17 340-359 155 155 155 276 276 276 F T T <--- 18 360-379 146 146 146 267 267 267 F F F 19 380-399 155 155 155 276 276 276 F T T <--- 20 400-419 155 155 155 276 276 276 F T T <--- 21 420-439 135 135 135 243 243 243 F T T <--- 22 440-459 149 149 149 269 269 269 F F F 23 460-479 154 154 154 275 275 275 F F F 24 480-499 153 153 153 273 273 273 F T T <--- 25 500-519 154 154 154 275 275 275 F T T <--- 26 520-539 154 154 154 275 275 275 F F F 27 540-559 155 155 155 276 276 276 F T T <--- 28 560-579 154 154 154 274 274 274 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 155, Max flush: 276 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 78 Verifying data now Files /tmp/test_repair/test_repair_verify.data and /tmp/test_repair/test_repair_verify.data.last differ 124a125 > 7, 126d126 < 5, diff found after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 274 274 274 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 156 156 156 277 277 277 F F F 3 060-079 154 154 154 274 274 274 F F F 4 080-099 156 156 156 277 277 277 F F F 5 100-119 154 154 154 275 275 275 F F F 6 120-139 156 156 156 277 277 277 F F F 7 140-159 154 154 154 274 274 274 F F F 8 160-179 152 152 152 272 272 272 F F F 9 180-199 156 156 156 277 277 277 F F F 10 200-219 156 156 156 277 277 277 F F F 11 220-239 153 153 153 273 273 273 F F F 12 240-259 154 154 154 275 275 275 F F F 13 260-279 156 156 156 277 277 277 F F F 14 280-299 154 154 154 274 274 274 F F F 15 300-319 156 156 156 277 277 277 F F F 16 320-339 154 154 154 274 274 274 F F F 17 340-359 156 156 156 277 277 277 F F F 18 360-379 146 146 146 267 267 267 F F F 19 380-399 156 156 156 277 277 277 F F F 20 400-419 156 156 156 277 277 277 F F F 21 420-439 156 156 156 277 277 277 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 154 154 154 275 275 275 F F F 24 480-499 156 156 156 277 277 277 F F F 25 500-519 156 156 156 277 277 277 F F F 26 520-539 154 154 154 275 275 275 F F F 27 540-559 156 156 156 277 277 277 F F F 28 560-579 154 154 154 274 274 274 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 156, Max flush: 277 Begin loop 79 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.160394778Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.168521396Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.168807966Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.168834964Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.168844034Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.168856023Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777} {"msg":"Crucible 52e96166-c41e-4011-9d82-0f43413edf2f has session id: cc72ce45-7a60-4626-b448-f455bd6d0110","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169218528Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"Upstairs opts: Upstairs UUID: 52e96166-c41e-4011-9d82-0f43413edf2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169247286Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"Crucible stats registered with UUID: 52e96166-c41e-4011-9d82-0f43413edf2f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169257955Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:27.169293243Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169473701Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"io task","client":"2","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.1694901Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"io task","client":"0","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169500929Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"io task","client":"1","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169671037Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169684666Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777} {"msg":"52e96166-c41e-4011-9d82-0f43413edf2f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169708365Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169719704Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"client":"0","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169729903Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"client":"1","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169740013Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"client":"2","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169749892Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"io task","client":"2","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169760381Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"io task","client":"0","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169770691Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"io task","client":"1","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"ds_connection connected from Ok(127.0.0.1:63056)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169849295Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"io task","client":"2","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"ds_connection connected from Ok(127.0.0.1:42716)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169860884Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"io task","client":"0","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"ds_connection connected from Ok(127.0.0.1:43133)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.169871614Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"io task","client":"1","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.170538748Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"client":"1","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.170587735Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"client":"2","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.170613693Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"client":"0","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.170913443Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.170923742Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"[0]R flush_numbers[0..12]: [274, 276, 277, 274, 277, 275, 277, 274, 272, 277, 277, 273]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.17096011Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"downstairs","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"[0]R generation[0..12]: [154, 155, 156, 154, 156, 154, 156, 154, 152, 156, 156, 153]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.170975389Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"downstairs","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.170986908Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"downstairs","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"[1]R flush_numbers[0..12]: [274, 276, 277, 274, 277, 275, 277, 274, 272, 277, 277, 273]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.170997927Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"downstairs","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"[1]R generation[0..12]: [154, 155, 156, 154, 156, 154, 156, 154, 152, 156, 156, 153]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171009136Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"downstairs","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171020475Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"downstairs","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"[2]R flush_numbers[0..12]: [274, 276, 277, 274, 277, 275, 277, 274, 272, 277, 277, 273]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171031365Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"downstairs","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"[2]R generation[0..12]: [154, 155, 156, 154, 156, 154, 156, 154, 152, 156, 156, 153]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171042134Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"downstairs","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171053443Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"downstairs","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"Max found gen is 157","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171063293Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"downstairs","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"Generation requested: 158 >= found:157","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171073532Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"downstairs","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171083461Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"downstairs","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171093351Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"":"downstairs","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.17110375Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171113259Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"client":"0","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171124458Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"client":"1","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171134478Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"client":"2","":"downstairs","session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171144727Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171154166Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"52e96166-c41e-4011-9d82-0f43413edf2f is now active with session: cc72ce45-7a60-4626-b448-f455bd6d0110","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171164436Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171173905Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171181834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"session_id":"cc72ce45-7a60-4626-b448-f455bd6d0110"} {"msg":"Activated sub_volume 52e96166-c41e-4011-9d82-0f43413edf2f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171191234Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.171199743Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.17153676Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"task":"crutest"} 01/30 Flush 02/30 Write block 295 len 24576 data: 12 13 15 17 15 14 03/30 Write block 242 len 32768 data: 18 17 15 13 16 12 8 8 04/30 Write block 283 len 32768 data: 8 7 6 6 9 8 7 5 05/30 Write block 155 len 16384 data: 10 13 9 6 06/30 Write block 467 len 8192 data: 19 19 07/30 Write block 405 len 40960 data: 11 10 10 10 7 8 11 11 11 11 08/30 Read block 103 len 20480 09/30 Read block 251 len 8192 10/30 Read block 148 len 8192 11/30 Read block 466 len 4096 12/30 Write block 409 len 40960 data: 8 9 12 12 12 12 13 15 13 12 13/30 Read block 98 len 36864 14/30 Write block 217 len 40960 data: 13 14 15 14 14 11 8 11 10 11 15/30 Read block 276 len 40960 16/30 Read block 292 len 16384 17/30 Read block 14 len 28672 18/30 Write block 182 len 32768 data: 9 11 9 11 10 9 13 13 19/30 Write block 237 len 20480 data: 9 9 11 14 17 20/30 Flush 21/30 Write block 398 len 24576 data: 17 18 16 15 16 13 22/30 Flush 23/30 Write block 253 len 36864 data: 9 11 11 12 13 15 15 13 12 24/30 Write block 363 len 40960 data: 13 12 14 15 13 14 15 14 12 15 25/30 Read block 433 len 8192 26/30 Read block 164 len 28672 27/30 Write block 488 len 20480 data: 13 12 10 11 11 28/30 Read block 212 len 24576 29/30 Read block 386 len 16384 30/30 Read block 446 len 12288 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 6 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Write 8 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Write 2 Done Done Done false 1006 Acked Write 10 Done Done Done false 1007 Acked Read 5 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 9 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Write 5 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Write 10 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Read 7 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 23 24 Done 29 28 6 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:27.179969777Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:27.180261967Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1777,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 274 274 274 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 156 156 156 277 277 277 F F F 3 060-079 154 154 154 274 274 274 F F F 4 080-099 156 156 156 277 277 277 F F F 5 100-119 154 154 154 275 275 275 F F F 6 120-139 156 156 156 277 277 277 F F F 7 140-159 158 158 154 279 279 274 F F T <--- 8 160-179 152 152 152 272 272 272 F F F 9 180-199 158 158 156 279 279 277 F F F <--- 10 200-219 158 158 156 279 279 277 F F F <--- 11 220-239 158 158 153 279 279 273 F F F <--- 12 240-259 158 158 154 279 279 275 T T T <--- 13 260-279 156 156 156 277 277 277 T T F <--- 14 280-299 158 158 154 279 279 274 F F T <--- 15 300-319 158 158 156 279 279 277 F F T <--- 16 320-339 154 154 154 274 274 274 F F F 17 340-359 156 156 156 277 277 277 F F F 18 360-379 146 146 146 267 267 267 T T F <--- 19 380-399 158 158 156 280 280 277 F F F <--- 20 400-419 158 158 156 280 280 277 F F T <--- 21 420-439 156 156 156 277 277 277 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 158 158 154 279 279 275 F F T <--- 24 480-499 156 156 156 277 277 277 T T F <--- 25 500-519 156 156 156 277 277 277 F F F 26 520-539 154 154 154 275 275 275 F F F 27 540-559 156 156 156 277 277 277 F F F 28 560-579 154 154 154 274 274 274 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 158, Max flush: 280 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 274 274 274 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 156 156 156 277 277 277 F F F 3 060-079 154 154 154 274 274 274 F F F 4 080-099 156 156 156 277 277 277 F F F 5 100-119 154 154 154 275 275 275 F F F 6 120-139 156 156 156 277 277 277 F F F 7 140-159 158 158 158 279 279 279 F F F 8 160-179 152 152 152 272 272 272 F F F 9 180-199 158 158 158 279 279 279 F F F 10 200-219 158 158 158 279 279 279 F F F 11 220-239 158 158 158 279 279 279 F F F 12 240-259 159 159 159 281 281 281 F F F 13 260-279 159 159 159 281 281 281 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 158 158 158 279 279 279 F F F 16 320-339 154 154 154 274 274 274 F F F 17 340-359 156 156 156 277 277 277 F F F 18 360-379 159 159 159 281 281 281 F F F 19 380-399 158 158 158 280 280 280 F F F 20 400-419 158 158 158 280 280 280 F F F 21 420-439 156 156 156 277 277 277 F F F 22 440-459 149 149 149 269 269 269 F F F 23 460-479 158 158 158 279 279 279 F F F 24 480-499 159 159 159 281 281 281 F F F 25 500-519 156 156 156 277 277 277 F F F 26 520-539 154 154 154 275 275 275 F F F 27 540-559 156 156 156 277 277 277 F F F 28 560-579 154 154 154 274 274 274 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 159, Max flush: 281 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.423702927Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.43204541Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.43233316Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.432358618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.432368198Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.432380047Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785} {"msg":"Crucible 8ab2685d-b01d-40c1-b5e1-ebfdd36f679f has session id: 0f2ca4a1-3bd6-4842-ad72-9b56cf035a18","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.432813447Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"Upstairs opts: Upstairs UUID: 8ab2685d-b01d-40c1-b5e1-ebfdd36f679f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.432842825Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"Crucible stats registered with UUID: 8ab2685d-b01d-40c1-b5e1-ebfdd36f679f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.432853745Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:31.432863434Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433022503Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"io task","client":"1","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433039672Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"io task","client":"0","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433050041Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"io task","client":"2","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433183952Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433202061Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785} {"msg":"8ab2685d-b01d-40c1-b5e1-ebfdd36f679f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433234359Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433244088Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"client":"0","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433254037Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"client":"1","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433264017Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"client":"2","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433273926Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"io task","client":"2","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433284375Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"io task","client":"1","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433346891Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"io task","client":"0","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"ds_connection connected from Ok(127.0.0.1:56099)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433373429Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"io task","client":"1","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"ds_connection connected from Ok(127.0.0.1:35787)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433385158Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"io task","client":"2","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"ds_connection connected from Ok(127.0.0.1:38283)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.433395978Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"io task","client":"0","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434041404Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"client":"0","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434062662Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"client":"2","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434146997Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"client":"1","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434401619Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434455146Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"[0]R flush_numbers[0..12]: [274, 276, 277, 274, 277, 275, 277, 279, 272, 279, 279, 279]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434522231Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"downstairs","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"[0]R generation[0..12]: [154, 155, 156, 154, 156, 154, 156, 158, 152, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.43453381Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"downstairs","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.43454544Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"downstairs","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"[1]R flush_numbers[0..12]: [274, 276, 277, 274, 277, 275, 277, 279, 272, 279, 279, 279]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434556479Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"downstairs","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"[1]R generation[0..12]: [154, 155, 156, 154, 156, 154, 156, 158, 152, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434567548Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"downstairs","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434580687Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"downstairs","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"[2]R flush_numbers[0..12]: [274, 276, 277, 274, 277, 275, 277, 279, 272, 279, 279, 279]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434591756Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"downstairs","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"[2]R generation[0..12]: [154, 155, 156, 154, 156, 154, 156, 158, 152, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434602526Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"downstairs","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434613725Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"downstairs","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434623514Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"downstairs","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434633544Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"downstairs","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434643143Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"downstairs","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434652862Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"":"downstairs","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434662922Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434672201Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"client":"0","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.43468214Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"client":"1","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.43469218Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"client":"2","":"downstairs","session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434702359Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434711848Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"8ab2685d-b01d-40c1-b5e1-ebfdd36f679f is now active with session: 0f2ca4a1-3bd6-4842-ad72-9b56cf035a18","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434722108Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434732487Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"session_id":"0f2ca4a1-3bd6-4842-ad72-9b56cf035a18"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434742006Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785} {"msg":"Activated sub_volume 8ab2685d-b01d-40c1-b5e1-ebfdd36f679f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434750286Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.434758685Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.435099802Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"task":"crutest"} 01/30 Write block 370 len 8192 data: 15 13 02/30 Read block 580 len 28672 03/30 Read block 190 len 20480 04/30 Read block 236 len 16384 05/30 Read block 358 len 8192 06/30 Write block 271 len 4096 data: 7 07/30 Read block 191 len 24576 08/30 Read block 560 len 32768 09/30 Flush 10/30 Write block 314 len 20480 data: 19 17 20 21 23 11/30 Write block 537 len 12288 data: 14 15 16 12/30 Flush 13/30 Write block 426 len 32768 data: 10 11 13 11 11 13 13 13 14/30 Write block 457 len 8192 data: 12 15 15/30 Read block 537 len 40960 16/30 Read block 284 len 36864 17/30 Write block 534 len 36864 data: 15 17 14 15 16 17 14 13 14 18/30 Read block 143 len 20480 19/30 Flush 20/30 Flush 21/30 Write block 528 len 20480 data: 14 14 13 13 15 22/30 Write block 145 len 28672 data: 17 18 17 16 17 18 19 23/30 Read block 81 len 12288 24/30 Write block 155 len 20480 data: 11 14 10 7 10 25/30 Write block 195 len 32768 data: 18 12 11 13 15 16 15 13 26/30 Write block 554 len 20480 data: 16 18 15 11 13 27/30 Write block 227 len 24576 data: 12 14 10 10 12 12 28/30 Read block 539 len 12288 29/30 Write block 343 len 20480 data: 14 15 13 9 9 30/30 Write block 61 len 12288 data: 9 8 8 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 7 Done Sent Done false 1002 Acked Read 5 Done Sent Done false 1003 Acked Read 4 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Write 9 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Write 5 Done Sent Done false 1024 Acked Write 8 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 29 2 33 Done 28 1 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:31.443346141Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:31.44364851Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1785,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 274 274 274 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 156 156 156 277 277 277 F F F 3 060-079 154 154 154 274 274 274 T F T <--- 4 080-099 156 156 156 277 277 277 F F F 5 100-119 154 154 154 275 275 275 F F F 6 120-139 156 156 156 277 277 277 F F F 7 140-159 158 158 158 279 279 279 T F T <--- 8 160-179 152 152 152 272 272 272 F F F 9 180-199 158 158 158 279 279 279 T F T <--- 10 200-219 158 158 158 279 279 279 T F T <--- 11 220-239 158 158 158 279 279 279 T F T <--- 12 240-259 159 159 159 281 281 281 F F F 13 260-279 160 159 160 282 281 282 F F F <--- 14 280-299 158 158 158 279 279 279 F F F 15 300-319 160 158 160 283 279 283 F F F <--- 16 320-339 154 154 154 274 274 274 F F F 17 340-359 156 156 156 277 277 277 T F T <--- 18 360-379 160 159 160 282 281 282 F T F <--- 19 380-399 158 158 158 280 280 280 F F F 20 400-419 158 158 158 280 280 280 F F F 21 420-439 160 156 160 284 277 284 F F F <--- 22 440-459 160 149 160 284 269 284 F F F <--- 23 460-479 158 158 158 279 279 279 F F F 24 480-499 159 159 159 281 281 281 F F F 25 500-519 156 156 156 277 277 277 F F F 26 520-539 160 154 160 284 275 284 T F T <--- 27 540-559 160 156 160 284 277 284 T F T <--- 28 560-579 154 154 154 274 274 274 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 160, Max flush: 284 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 274 274 274 F F F 1 020-039 155 155 155 276 276 276 F F F 2 040-059 156 156 156 277 277 277 F F F 3 060-079 161 161 161 285 285 285 F F F 4 080-099 156 156 156 277 277 277 F F F 5 100-119 154 154 154 275 275 275 F F F 6 120-139 156 156 156 277 277 277 F F F 7 140-159 161 161 161 285 285 285 F F F 8 160-179 152 152 152 272 272 272 F F F 9 180-199 161 161 161 285 285 285 F F F 10 200-219 161 161 161 285 285 285 F F F 11 220-239 161 161 161 285 285 285 F F F 12 240-259 159 159 159 281 281 281 F F F 13 260-279 160 160 160 282 282 282 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 160 160 160 283 283 283 F F F 16 320-339 154 154 154 274 274 274 F F F 17 340-359 161 161 161 285 285 285 F F F 18 360-379 160 160 160 282 282 282 F F F 19 380-399 158 158 158 280 280 280 F F F 20 400-419 158 158 158 280 280 280 F F F 21 420-439 160 160 160 284 284 284 F F F 22 440-459 160 160 160 284 284 284 F F F 23 460-479 158 158 158 279 279 279 F F F 24 480-499 159 159 159 281 281 281 F F F 25 500-519 156 156 156 277 277 277 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 161 161 161 285 285 285 F F F 28 560-579 154 154 154 274 274 274 F F F 29 580-599 146 146 146 267 267 267 F F F Max gen: 161, Max flush: 285 Begin loop 81 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.693081513Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.701442034Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.701745284Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.701770732Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.701779711Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.70179174Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793} {"msg":"Crucible 904961bd-c2b7-4668-ba40-b7767d5161c7 has session id: dfd3d515-ade3-41d3-88db-22899b0bd3ee","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702191533Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"Upstairs opts: Upstairs UUID: 904961bd-c2b7-4668-ba40-b7767d5161c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702222451Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"Crucible stats registered with UUID: 904961bd-c2b7-4668-ba40-b7767d5161c7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.70223423Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:35.70224425Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702407748Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"io task","client":"1","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702424777Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"io task","client":"0","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702435207Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"io task","client":"2","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702588076Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702601625Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793} {"msg":"904961bd-c2b7-4668-ba40-b7767d5161c7 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702610245Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702619214Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"client":"0","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702629243Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"client":"1","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702639123Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"client":"2","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702648902Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"io task","client":"2","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702659191Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"io task","client":"0","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702669241Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"io task","client":"1","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:46523)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702758885Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"io task","client":"1","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:51286)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702785313Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"io task","client":"0","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:46334)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.702796792Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"io task","client":"2","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.703504864Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"client":"0","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.703532732Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"client":"1","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.703544681Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"client":"2","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.703846351Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.70385923Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"[0]R flush_numbers[0..12]: [274, 276, 277, 285, 277, 275, 277, 285, 272, 285, 285, 285]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.703893737Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"downstairs","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"[0]R generation[0..12]: [154, 155, 156, 161, 156, 154, 156, 161, 152, 161, 161, 161]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.703919616Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"downstairs","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.703931985Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"downstairs","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"[1]R flush_numbers[0..12]: [274, 276, 277, 285, 277, 275, 277, 285, 272, 285, 285, 285]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.703943204Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"downstairs","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"[1]R generation[0..12]: [154, 155, 156, 161, 156, 154, 156, 161, 152, 161, 161, 161]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.703954213Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"downstairs","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.703965532Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"downstairs","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"[2]R flush_numbers[0..12]: [274, 276, 277, 285, 277, 275, 277, 285, 272, 285, 285, 285]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.703976452Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"downstairs","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"[2]R generation[0..12]: [154, 155, 156, 161, 156, 154, 156, 161, 152, 161, 161, 161]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.703987201Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"downstairs","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704010449Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"downstairs","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704020279Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"downstairs","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704033208Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"downstairs","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704043107Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"downstairs","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704052917Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"":"downstairs","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704062906Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704071915Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"client":"0","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704081745Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"client":"1","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704091764Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"client":"2","":"downstairs","session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704101903Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704111293Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"904961bd-c2b7-4668-ba40-b7767d5161c7 is now active with session: dfd3d515-ade3-41d3-88db-22899b0bd3ee","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704121472Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704130741Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704138551Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"session_id":"dfd3d515-ade3-41d3-88db-22899b0bd3ee"} {"msg":"Activated sub_volume 904961bd-c2b7-4668-ba40-b7767d5161c7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704156709Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.704453879Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"task":"crutest"} 01/30 Write block 414 len 28672 data: 13 14 16 14 13 12 13 02/30 Flush 03/30 Write block 581 len 24576 data: 11 9 9 8 10 9 04/30 Write block 185 len 28672 data: 12 11 10 14 14 13 13 05/30 Flush 06/30 Write block 219 len 12288 data: 16 15 15 07/30 Read block 421 len 16384 08/30 Write block 565 len 20480 data: 14 11 13 12 14 09/30 Read block 53 len 20480 10/30 Read block 84 len 4096 11/30 Read block 264 len 12288 12/30 Read block 521 len 12288 13/30 Read block 308 len 16384 14/30 Read block 491 len 20480 15/30 Read block 136 len 28672 16/30 Read block 501 len 16384 17/30 Read block 328 len 20480 18/30 Read block 564 len 36864 19/30 Write block 120 len 36864 data: 9 7 7 6 9 13 15 11 12 20/30 Write block 237 len 32768 data: 10 10 12 15 18 19 18 16 21/30 Read block 33 len 4096 22/30 Read block 363 len 24576 23/30 Read block 325 len 36864 24/30 Read block 257 len 32768 25/30 Write block 316 len 28672 data: 21 22 24 21 17 15 14 26/30 Flush 27/30 Write block 37 len 36864 data: 13 13 14 14 12 12 11 12 13 28/30 Read block 79 len 12288 29/30 Write block 120 len 32768 data: 10 8 8 7 10 14 16 12 30/30 Read block 224 len 24576 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Sent Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Read 4 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Read 4 Done Sent Done false 1013 Acked Read 5 Done Sent Done false 1014 Acked Read 7 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Read 9 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 8 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:35.713941844Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:35.714239473Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1793,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 274 274 274 F F F 1 020-039 155 155 155 276 276 276 T F T <--- 2 040-059 156 156 156 277 277 277 T F T <--- 3 060-079 161 161 161 285 285 285 F F F 4 080-099 156 156 156 277 277 277 F F F 5 100-119 154 154 154 275 275 275 F F F 6 120-139 162 156 162 288 277 288 T F T <--- 7 140-159 161 161 161 285 285 285 F F F 8 160-179 152 152 152 272 272 272 F F F 9 180-199 162 161 162 287 285 287 F F F <--- 10 200-219 162 161 162 288 285 288 F F F <--- 11 220-239 162 161 162 288 285 288 F F F <--- 12 240-259 162 159 162 288 281 288 F F F <--- 13 260-279 160 160 160 282 282 282 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 162 160 162 288 283 288 F F F <--- 16 320-339 162 154 162 288 274 288 F F F <--- 17 340-359 161 161 161 285 285 285 F F F 18 360-379 160 160 160 282 282 282 F F F 19 380-399 158 158 158 280 280 280 F F F 20 400-419 162 158 162 286 280 286 F F F <--- 21 420-439 162 160 162 286 284 286 F F F <--- 22 440-459 160 160 160 284 284 284 F F F 23 460-479 158 158 158 279 279 279 F F F 24 480-499 159 159 159 281 281 281 F F F 25 500-519 156 156 156 277 277 277 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 161 161 161 285 285 285 F F F 28 560-579 162 154 162 288 274 288 F F F <--- 29 580-599 162 146 162 287 267 287 F F F <--- Max gen: 162, Max flush: 288 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 274 274 274 F F F 1 020-039 163 163 163 289 289 289 F F F 2 040-059 163 163 163 289 289 289 F F F 3 060-079 161 161 161 285 285 285 F F F 4 080-099 156 156 156 277 277 277 F F F 5 100-119 154 154 154 275 275 275 F F F 6 120-139 163 163 163 289 289 289 F F F 7 140-159 161 161 161 285 285 285 F F F 8 160-179 152 152 152 272 272 272 F F F 9 180-199 162 162 162 287 287 287 F F F 10 200-219 162 162 162 288 288 288 F F F 11 220-239 162 162 162 288 288 288 F F F 12 240-259 162 162 162 288 288 288 F F F 13 260-279 160 160 160 282 282 282 F F F 14 280-299 158 158 158 279 279 279 F F F 15 300-319 162 162 162 288 288 288 F F F 16 320-339 162 162 162 288 288 288 F F F 17 340-359 161 161 161 285 285 285 F F F 18 360-379 160 160 160 282 282 282 F F F 19 380-399 158 158 158 280 280 280 F F F 20 400-419 162 162 162 286 286 286 F F F 21 420-439 162 162 162 286 286 286 F F F 22 440-459 160 160 160 284 284 284 F F F 23 460-479 158 158 158 279 279 279 F F F 24 480-499 159 159 159 281 281 281 F F F 25 500-519 156 156 156 277 277 277 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 161 161 161 285 285 285 F F F 28 560-579 162 162 162 288 288 288 F F F 29 580-599 162 162 162 287 287 287 F F F Max gen: 163, Max flush: 289 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.956189215Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.964304683Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.964587364Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.964614312Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.964623302Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.964635041Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801} {"msg":"Crucible 7bdf7d60-cc36-4083-9893-05f120d29df4 has session id: 88e0ace1-1f0d-4e4f-a05f-8887826eb4ca","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.964967798Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"Upstairs opts: Upstairs UUID: 7bdf7d60-cc36-4083-9893-05f120d29df4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965063802Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"Crucible stats registered with UUID: 7bdf7d60-cc36-4083-9893-05f120d29df4","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965074731Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:39.96508406Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965253739Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"io task","client":"0","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965273667Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"io task","client":"1","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965285467Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"io task","client":"2","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965443346Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965461455Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801} {"msg":"7bdf7d60-cc36-4083-9893-05f120d29df4 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965484793Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965496702Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"client":"0","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965506592Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"client":"1","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965516601Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"client":"2","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.96552651Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"io task","client":"2","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965552288Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"io task","client":"1","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965601865Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"io task","client":"0","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:39900)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965612924Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"io task","client":"1","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:56258)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965623704Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"io task","client":"2","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"ds_connection connected from Ok(127.0.0.1:58124)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.965774613Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"io task","client":"0","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966237762Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"client":"0","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.96626468Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"client":"1","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966357224Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"client":"2","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966531152Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966548291Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"[0]R flush_numbers[0..12]: [274, 289, 289, 285, 277, 275, 289, 285, 272, 287, 288, 288]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966626755Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"downstairs","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"[0]R generation[0..12]: [154, 163, 163, 161, 156, 154, 163, 161, 152, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966640974Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"downstairs","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966652294Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"downstairs","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"[1]R flush_numbers[0..12]: [274, 289, 289, 285, 277, 275, 289, 285, 272, 287, 288, 288]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966663383Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"downstairs","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"[1]R generation[0..12]: [154, 163, 163, 161, 156, 154, 163, 161, 152, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966674142Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"downstairs","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966685311Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"downstairs","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"[2]R flush_numbers[0..12]: [274, 289, 289, 285, 277, 275, 289, 285, 272, 287, 288, 288]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966696221Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"downstairs","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"[2]R generation[0..12]: [154, 163, 163, 161, 156, 154, 163, 161, 152, 162, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966707Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"downstairs","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966718489Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"downstairs","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966728128Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"downstairs","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966738198Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"downstairs","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"Next flush: 290","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966747997Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"downstairs","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966757746Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"":"downstairs","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966767846Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966776875Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"client":"0","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966788134Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"client":"1","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966797994Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"client":"2","":"downstairs","session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966808123Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966817552Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"7bdf7d60-cc36-4083-9893-05f120d29df4 is now active with session: 88e0ace1-1f0d-4e4f-a05f-8887826eb4ca","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966827762Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966837321Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.96684519Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"session_id":"88e0ace1-1f0d-4e4f-a05f-8887826eb4ca"} {"msg":"Activated sub_volume 7bdf7d60-cc36-4083-9893-05f120d29df4","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.9668548Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.966863449Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.967196467Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"task":"crutest"} 01/30 Write block 116 len 28672 data: 9 8 10 11 11 9 9 02/30 Write block 132 len 40960 data: 14 8 11 9 12 14 16 17 15 16 03/30 Read block 186 len 40960 04/30 Write block 322 len 32768 data: 15 15 15 11 10 12 14 16 05/30 Read block 344 len 24576 06/30 Read block 255 len 28672 07/30 Read block 285 len 16384 08/30 Write block 348 len 24576 data: 7 14 16 16 14 13 09/30 Read block 417 len 20480 10/30 Read block 438 len 4096 11/30 Flush 12/30 Write block 507 len 32768 data: 15 13 15 15 13 9 11 11 13/30 Read block 212 len 4096 14/30 Flush 15/30 Write block 558 len 32768 data: 14 15 14 15 14 16 15 15 16/30 Read block 481 len 4096 17/30 Write block 132 len 32768 data: 15 9 12 10 13 15 17 18 18/30 Read block 112 len 8192 19/30 Read block 206 len 32768 20/30 Flush 21/30 Write block 317 len 32768 data: 23 25 22 18 16 16 16 16 22/30 Flush 23/30 Write block 476 len 32768 data: 9 9 13 11 10 12 13 13 24/30 Read block 11 len 32768 25/30 Write block 315 len 20480 data: 18 22 24 26 23 26/30 Read block 505 len 16384 27/30 Write block 297 len 20480 data: 16 18 16 15 12 28/30 Read block 547 len 4096 29/30 Write block 179 len 12288 data: 8 6 7 30/30 Write block 126 len 12288 data: 17 13 13 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Sent false 1001 Acked Write 10 Done Done Sent false 1002 Acked Read 10 Done Done Sent false 1003 Acked Write 8 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Read 7 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Write 6 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Read 1 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 8 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Write 8 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 30 34 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:39.976807013Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:39.977091463Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1801,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 274 274 274 F F F 1 020-039 163 163 163 289 289 289 F F F 2 040-059 163 163 163 289 289 289 F F F 3 060-079 161 161 161 285 285 285 F F F 4 080-099 156 156 156 277 277 277 F F F 5 100-119 164 164 154 290 290 275 F F F <--- 6 120-139 164 164 163 292 292 289 T T F <--- 7 140-159 164 164 161 290 290 285 F F F <--- 8 160-179 152 152 152 272 272 272 T T F <--- 9 180-199 162 162 162 287 287 287 T T F <--- 10 200-219 162 162 162 288 288 288 F F F 11 220-239 162 162 162 288 288 288 F F F 12 240-259 162 162 162 288 288 288 F F F 13 260-279 160 160 160 282 282 282 F F F 14 280-299 158 158 158 279 279 279 T T F <--- 15 300-319 164 164 162 293 293 288 T T F <--- 16 320-339 164 164 162 293 293 288 F F F <--- 17 340-359 164 164 161 290 290 285 F F F <--- 18 360-379 160 160 160 282 282 282 F F F 19 380-399 158 158 158 280 280 280 F F F 20 400-419 162 162 162 286 286 286 F F F 21 420-439 162 162 162 286 286 286 F F F 22 440-459 160 160 160 284 284 284 F F F 23 460-479 158 158 158 279 279 279 T T F <--- 24 480-499 159 159 159 281 281 281 T T F <--- 25 500-519 164 164 156 291 291 277 F F F <--- 26 520-539 161 161 161 285 285 285 F F F 27 540-559 164 164 161 292 292 285 F F F <--- 28 560-579 164 164 162 292 292 288 F F F <--- 29 580-599 162 162 162 287 287 287 F F F Max gen: 164, Max flush: 293 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 154 154 154 274 274 274 F F F 1 020-039 163 163 163 289 289 289 F F F 2 040-059 163 163 163 289 289 289 F F F 3 060-079 161 161 161 285 285 285 F F F 4 080-099 156 156 156 277 277 277 F F F 5 100-119 164 164 164 290 290 290 F F F 6 120-139 165 165 165 294 294 294 F F F 7 140-159 164 164 164 290 290 290 F F F 8 160-179 165 165 165 294 294 294 F F F 9 180-199 165 165 165 294 294 294 F F F 10 200-219 162 162 162 288 288 288 F F F 11 220-239 162 162 162 288 288 288 F F F 12 240-259 162 162 162 288 288 288 F F F 13 260-279 160 160 160 282 282 282 F F F 14 280-299 165 165 165 294 294 294 F F F 15 300-319 165 165 165 294 294 294 F F F 16 320-339 164 164 164 293 293 293 F F F 17 340-359 164 164 164 290 290 290 F F F 18 360-379 160 160 160 282 282 282 F F F 19 380-399 158 158 158 280 280 280 F F F 20 400-419 162 162 162 286 286 286 F F F 21 420-439 162 162 162 286 286 286 F F F 22 440-459 160 160 160 284 284 284 F F F 23 460-479 165 165 165 294 294 294 F F F 24 480-499 165 165 165 294 294 294 F F F 25 500-519 164 164 164 291 291 291 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 164 164 164 292 292 292 F F F 28 560-579 164 164 164 292 292 292 F F F 29 580-599 162 162 162 287 287 287 F F F Max gen: 165, Max flush: 294 Begin loop 83 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.227887753Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.236175959Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.23646518Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.236492448Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.236501607Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.236513466Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809} {"msg":"Crucible be6f88bf-a9b0-4b26-bc46-649c989dbaff has session id: b0462f47-1d1e-4c16-bde3-3e6d474b4784","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.236850313Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"Upstairs opts: Upstairs UUID: be6f88bf-a9b0-4b26-bc46-649c989dbaff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.236870562Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"Crucible stats registered with UUID: be6f88bf-a9b0-4b26-bc46-649c989dbaff","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.236880941Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:44.23689501Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237072178Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"io task","client":"2","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237089347Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"io task","client":"0","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237099767Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"io task","client":"1","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237226948Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237240957Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809} {"msg":"be6f88bf-a9b0-4b26-bc46-649c989dbaff active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237257576Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237266785Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"client":"0","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237276834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"client":"1","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237286814Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"client":"2","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237297003Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"io task","client":"2","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237316342Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"io task","client":"1","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237377978Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"io task","client":"0","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"ds_connection connected from Ok(127.0.0.1:61156)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237405626Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"io task","client":"2","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"ds_connection connected from Ok(127.0.0.1:52626)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237421835Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"io task","client":"1","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"ds_connection connected from Ok(127.0.0.1:33689)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.237511429Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"io task","client":"0","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238048332Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"client":"0","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238170704Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"client":"1","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238235199Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"client":"2","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238401268Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238454834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"[0]R flush_numbers[0..12]: [274, 289, 289, 285, 277, 290, 294, 290, 294, 294, 288, 288]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238539979Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"downstairs","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"[0]R generation[0..12]: [154, 163, 163, 161, 156, 164, 165, 164, 165, 165, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238551728Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"downstairs","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238563067Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"downstairs","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"[1]R flush_numbers[0..12]: [274, 289, 289, 285, 277, 290, 294, 290, 294, 294, 288, 288]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238574006Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"downstairs","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"[1]R generation[0..12]: [154, 163, 163, 161, 156, 164, 165, 164, 165, 165, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238584865Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"downstairs","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238596055Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"downstairs","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"[2]R flush_numbers[0..12]: [274, 289, 289, 285, 277, 290, 294, 290, 294, 294, 288, 288]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238608544Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"downstairs","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"[2]R generation[0..12]: [154, 163, 163, 161, 156, 164, 165, 164, 165, 165, 162, 162]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238619353Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"downstairs","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238630422Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"downstairs","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238640282Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"downstairs","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238650411Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"downstairs","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.23866009Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"downstairs","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.23867006Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"":"downstairs","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238680169Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238689268Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"client":"0","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238699598Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"client":"1","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238709817Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"client":"2","":"downstairs","session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238720206Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238729656Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"be6f88bf-a9b0-4b26-bc46-649c989dbaff is now active with session: b0462f47-1d1e-4c16-bde3-3e6d474b4784","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238739895Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238749404Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238759084Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"session_id":"b0462f47-1d1e-4c16-bde3-3e6d474b4784"} {"msg":"Activated sub_volume be6f88bf-a9b0-4b26-bc46-649c989dbaff","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238768623Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.238777092Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.239124359Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"task":"crutest"} 01/30 Write block 580 len 36864 data: 15 12 10 10 9 11 10 11 7 02/30 Read block 390 len 16384 03/30 Write block 440 len 40960 data: 12 12 14 9 8 7 5 6 8 10 04/30 Write block 377 len 12288 data: 14 17 15 05/30 Flush 06/30 Write block 339 len 4096 data: 14 07/30 Write block 206 len 4096 data: 15 08/30 Read block 255 len 28672 09/30 Read block 515 len 32768 10/30 Read block 426 len 32768 11/30 Read block 70 len 24576 12/30 Write block 54 len 4096 data: 9 13/30 Read block 518 len 12288 14/30 Read block 148 len 32768 15/30 Write block 391 len 12288 data: 15 14 14 16/30 Read block 555 len 36864 17/30 Read block 492 len 8192 18/30 Flush 19/30 Write block 4 len 4096 data: 10 20/30 Flush 21/30 Write block 381 len 32768 data: 13 14 13 11 13 12 9 10 22/30 Write block 369 len 8192 data: 16 16 23/30 Write block 219 len 12288 data: 17 16 16 24/30 Write block 417 len 36864 data: 15 14 13 14 10 10 9 8 9 25/30 Write block 582 len 4096 data: 11 26/30 Write block 161 len 20480 data: 12 13 13 14 15 27/30 Read block 526 len 16384 28/30 Read block 28 len 32768 29/30 Write block 67 len 28672 data: 5 6 8 6 6 8 9 30/30 Write block 87 len 8192 data: 7 8 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 8 Done Sent Done false 1010 Acked Read 6 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 9 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 8 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Read 8 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 26 2 30 Done 28 4 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:44.246778428Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:44.247070078Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1809,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 154 166 297 274 297 F F F <--- 1 020-039 163 163 163 289 289 289 F F F 2 040-059 166 163 166 296 289 296 F F F <--- 3 060-079 161 161 161 285 285 285 T F T <--- 4 080-099 156 156 156 277 277 277 T F T <--- 5 100-119 164 164 164 290 290 290 F F F 6 120-139 165 165 165 294 294 294 F F F 7 140-159 164 164 164 290 290 290 F F F 8 160-179 165 165 165 294 294 294 T F T <--- 9 180-199 165 165 165 294 294 294 F F F 10 200-219 166 162 166 296 288 296 T F T <--- 11 220-239 162 162 162 288 288 288 T F T <--- 12 240-259 162 162 162 288 288 288 F F F 13 260-279 160 160 160 282 282 282 F F F 14 280-299 165 165 165 294 294 294 F F F 15 300-319 165 165 165 294 294 294 F F F 16 320-339 166 164 166 296 293 296 F F F <--- 17 340-359 164 164 164 290 290 290 F F F 18 360-379 166 160 166 295 282 295 T T T <--- 19 380-399 166 158 166 296 280 296 T F T <--- 20 400-419 162 162 162 286 286 286 T F T <--- 21 420-439 162 162 162 286 286 286 T F T <--- 22 440-459 166 160 166 295 284 295 F T F <--- 23 460-479 165 165 165 294 294 294 F F F 24 480-499 165 165 165 294 294 294 F F F 25 500-519 164 164 164 291 291 291 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 164 164 164 292 292 292 F F F 28 560-579 164 164 164 292 292 292 F F F 29 580-599 166 162 166 295 287 295 T T T <--- Max gen: 166, Max flush: 297 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 297 297 297 F F F 1 020-039 163 163 163 289 289 289 F F F 2 040-059 166 166 166 296 296 296 F F F 3 060-079 167 167 167 298 298 298 F F F 4 080-099 167 167 167 298 298 298 F F F 5 100-119 164 164 164 290 290 290 F F F 6 120-139 165 165 165 294 294 294 F F F 7 140-159 164 164 164 290 290 290 F F F 8 160-179 167 167 167 298 298 298 F F F 9 180-199 165 165 165 294 294 294 F F F 10 200-219 167 167 167 298 298 298 F F F 11 220-239 167 167 167 298 298 298 F F F 12 240-259 162 162 162 288 288 288 F F F 13 260-279 160 160 160 282 282 282 F F F 14 280-299 165 165 165 294 294 294 F F F 15 300-319 165 165 165 294 294 294 F F F 16 320-339 166 166 166 296 296 296 F F F 17 340-359 164 164 164 290 290 290 F F F 18 360-379 167 167 167 298 298 298 F F F 19 380-399 167 167 167 298 298 298 F F F 20 400-419 167 167 167 298 298 298 F F F 21 420-439 167 167 167 298 298 298 F F F 22 440-459 166 166 166 295 295 295 F F F 23 460-479 165 165 165 294 294 294 F F F 24 480-499 165 165 165 294 294 294 F F F 25 500-519 164 164 164 291 291 291 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 164 164 164 292 292 292 F F F 28 560-579 164 164 164 292 292 292 F F F 29 580-599 167 167 167 298 298 298 F F F Max gen: 167, Max flush: 298 Begin loop 84 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.496354861Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:48.503553791Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.503617977Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.510566434Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.510812907Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.510836356Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.510845425Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.510857474Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817} {"msg":"Crucible 4c8b8e0d-7827-4cac-885f-0ea5d19c5d33 has session id: 7512032f-6d43-49de-a285-6778e736c0c3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511200471Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"Upstairs opts: Upstairs UUID: 4c8b8e0d-7827-4cac-885f-0ea5d19c5d33, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511230109Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"Crucible stats registered with UUID: 4c8b8e0d-7827-4cac-885f-0ea5d19c5d33","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511243118Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:48.511252747Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511408717Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"io task","client":"2","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511426376Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"io task","client":"0","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511436975Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"io task","client":"1","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511587175Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511605423Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817} {"msg":"4c8b8e0d-7827-4cac-885f-0ea5d19c5d33 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511638701Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511648441Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"client":"0","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.51165845Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"client":"1","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511668749Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"client":"2","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511678658Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"io task","client":"2","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511689318Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"io task","client":"1","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511750924Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"io task","client":"0","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:63440)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.51180602Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"io task","client":"2","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43698)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511820999Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"io task","client":"0","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:45330)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.511832378Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"io task","client":"1","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512401599Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"client":"2","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512486674Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"client":"1","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512500043Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"client":"0","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512732767Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512774894Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"[0]R flush_numbers[0..12]: [297, 289, 296, 298, 298, 290, 294, 290, 298, 294, 298, 298]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.5128332Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"downstairs","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"[0]R generation[0..12]: [166, 163, 166, 167, 167, 164, 165, 164, 167, 165, 167, 167]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512845549Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"downstairs","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512857088Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"downstairs","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"[1]R flush_numbers[0..12]: [297, 289, 296, 298, 298, 290, 294, 290, 298, 294, 298, 298]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512868148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"downstairs","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"[1]R generation[0..12]: [166, 163, 166, 167, 167, 164, 165, 164, 167, 165, 167, 167]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512879277Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"downstairs","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512890666Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"downstairs","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"[2]R flush_numbers[0..12]: [297, 289, 296, 298, 298, 290, 294, 290, 298, 294, 298, 298]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512901735Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"downstairs","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"[2]R generation[0..12]: [166, 163, 166, 167, 167, 164, 165, 164, 167, 165, 167, 167]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512912755Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"downstairs","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512924094Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"downstairs","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512935923Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"downstairs","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512946212Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"downstairs","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512956002Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"downstairs","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512965871Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"":"downstairs","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.51297602Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.51298535Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"client":"0","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.512995499Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"client":"1","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.513005718Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"client":"2","":"downstairs","session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.513016177Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.513025697Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"4c8b8e0d-7827-4cac-885f-0ea5d19c5d33 is now active with session: 7512032f-6d43-49de-a285-6778e736c0c3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.513035966Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.513045555Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.513053635Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"session_id":"7512032f-6d43-49de-a285-6778e736c0c3"} {"msg":"Activated sub_volume 4c8b8e0d-7827-4cac-885f-0ea5d19c5d33","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.513063244Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.513071814Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.513402071Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"task":"crutest"} 01/30 Flush 02/30 Write block 86 len 4096 data: 7 03/30 Write block 259 len 24576 data: 16 14 13 10 12 10 04/30 Read block 277 len 12288 05/30 Flush 06/30 Write block 98 len 32768 data: 15 13 10 10 9 9 9 8 07/30 Flush 08/30 Flush 09/30 Write block 72 len 12288 data: 9 10 10 10/30 Read block 531 len 4096 11/30 Flush 12/30 Write block 217 len 4096 data: 14 13/30 Write block 588 len 16384 data: 8 8 10 14 14/30 Flush 15/30 Write block 133 len 32768 data: 10 13 11 14 16 18 19 16 16/30 Write block 199 len 24576 data: 16 17 16 14 12 16 17/30 Read block 182 len 12288 18/30 Read block 438 len 36864 19/30 Read block 288 len 32768 20/30 Read block 334 len 36864 21/30 Write block 398 len 24576 data: 18 19 17 16 17 14 22/30 Write block 199 len 24576 data: 17 18 17 15 13 17 23/30 Read block 532 len 16384 24/30 Read block 546 len 28672 25/30 Read block 252 len 28672 26/30 Read block 5 len 4096 27/30 Read block 318 len 12288 28/30 Write block 330 len 4096 data: 14 29/30 Read block 219 len 32768 30/30 Write block 60 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Sent Done Done false 1001 Acked Write 1 Sent Done Done false 1002 Acked Write 6 Sent Done Done false 1003 Acked Read 3 Sent Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Write 6 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Read 7 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 1 1 32 Done 0 29 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:48.521925321Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:48.522221761Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1817,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 297 297 297 F F F 1 020-039 163 163 163 289 289 289 F F F 2 040-059 166 166 166 296 296 296 F F F 3 060-079 167 168 168 298 303 303 F T T <--- 4 080-099 167 168 168 298 301 301 F F F <--- 5 100-119 164 168 168 290 301 301 F F F <--- 6 120-139 165 165 165 294 294 294 F T T <--- 7 140-159 164 164 164 290 290 290 F T T <--- 8 160-179 167 167 167 298 298 298 F F F 9 180-199 165 165 165 294 294 294 F T T <--- 10 200-219 167 168 168 298 304 304 F T T <--- 11 220-239 167 167 167 298 298 298 F F F 12 240-259 162 168 168 288 300 300 F F F <--- 13 260-279 160 168 168 282 300 300 F F F <--- 14 280-299 165 165 165 294 294 294 F F F 15 300-319 165 165 165 294 294 294 F F F 16 320-339 166 166 166 296 296 296 F T T <--- 17 340-359 164 164 164 290 290 290 F F F 18 360-379 167 167 167 298 298 298 F F F 19 380-399 167 167 167 298 298 298 F T T <--- 20 400-419 167 167 167 298 298 298 F T T <--- 21 420-439 167 167 167 298 298 298 F F F 22 440-459 166 166 166 295 295 295 F F F 23 460-479 165 165 165 294 294 294 F F F 24 480-499 165 165 165 294 294 294 F F F 25 500-519 164 164 164 291 291 291 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 164 164 164 292 292 292 F F F 28 560-579 164 164 164 292 292 292 F F F 29 580-599 167 168 168 298 304 304 F F F <--- Max gen: 168, Max flush: 304 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 297 297 297 F F F 1 020-039 163 163 163 289 289 289 F F F 2 040-059 166 166 166 296 296 296 F F F 3 060-079 169 169 169 305 305 305 F F F 4 080-099 168 168 168 301 301 301 F F F 5 100-119 168 168 168 301 301 301 F F F 6 120-139 169 169 169 305 305 305 F F F 7 140-159 169 169 169 305 305 305 F F F 8 160-179 167 167 167 298 298 298 F F F 9 180-199 169 169 169 305 305 305 F F F 10 200-219 169 169 169 305 305 305 F F F 11 220-239 167 167 167 298 298 298 F F F 12 240-259 168 168 168 300 300 300 F F F 13 260-279 168 168 168 300 300 300 F F F 14 280-299 165 165 165 294 294 294 F F F 15 300-319 165 165 165 294 294 294 F F F 16 320-339 169 169 169 305 305 305 F F F 17 340-359 164 164 164 290 290 290 F F F 18 360-379 167 167 167 298 298 298 F F F 19 380-399 169 169 169 305 305 305 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 167 167 167 298 298 298 F F F 22 440-459 166 166 166 295 295 295 F F F 23 460-479 165 165 165 294 294 294 F F F 24 480-499 165 165 165 294 294 294 F F F 25 500-519 164 164 164 291 291 291 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 164 164 164 292 292 292 F F F 28 560-579 164 164 164 292 292 292 F F F 29 580-599 168 168 168 304 304 304 F F F Max gen: 169, Max flush: 305 Begin loop 85 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.772758829Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.781192605Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.781483885Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.781510843Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.781519673Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.781531522Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825} {"msg":"Crucible ab278692-fb1e-4e20-8eea-4b5302d99321 has session id: c3cbbad0-a1b8-4f74-9db4-9c8973c6310c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.781908036Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"Upstairs opts: Upstairs UUID: ab278692-fb1e-4e20-8eea-4b5302d99321, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.781937454Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"Crucible stats registered with UUID: ab278692-fb1e-4e20-8eea-4b5302d99321","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.781948584Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:52.781958433Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782130431Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"io task","client":"2","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.78214772Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"io task","client":"0","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782158449Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"io task","client":"1","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782284971Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782303559Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825} {"msg":"ab278692-fb1e-4e20-8eea-4b5302d99321 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782342747Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782352536Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"client":"0","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782362735Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"client":"1","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782372715Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"client":"2","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782382604Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"io task","client":"2","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782393353Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"io task","client":"1","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"ds_connection connected from Ok(127.0.0.1:56141)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782431031Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"io task","client":"2","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782457399Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"io task","client":"0","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36937)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782468948Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"io task","client":"1","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"ds_connection connected from Ok(127.0.0.1:51446)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.782557262Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"io task","client":"0","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783148302Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"client":"1","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783190079Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"client":"0","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783220457Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"client":"2","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783478979Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783490249Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"[0]R flush_numbers[0..12]: [297, 289, 296, 305, 301, 301, 305, 305, 298, 305, 305, 298]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783525916Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"downstairs","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"[0]R generation[0..12]: [166, 163, 166, 169, 168, 168, 169, 169, 167, 169, 169, 167]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783552375Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"downstairs","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783564734Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"downstairs","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"[1]R flush_numbers[0..12]: [297, 289, 296, 305, 301, 301, 305, 305, 298, 305, 305, 298]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783575783Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"downstairs","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"[1]R generation[0..12]: [166, 163, 166, 169, 168, 168, 169, 169, 167, 169, 169, 167]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783586602Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"downstairs","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783597881Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"downstairs","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"[2]R flush_numbers[0..12]: [297, 289, 296, 305, 301, 301, 305, 305, 298, 305, 305, 298]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783608911Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"downstairs","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"[2]R generation[0..12]: [166, 163, 166, 169, 168, 168, 169, 169, 167, 169, 169, 167]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.7836198Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"downstairs","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783631259Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"downstairs","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783641168Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"downstairs","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783651448Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"downstairs","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783661147Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"downstairs","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783670806Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"":"downstairs","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783681126Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783690295Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"client":"0","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783702844Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"client":"1","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783713184Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"client":"2","":"downstairs","session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783723763Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783733352Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"ab278692-fb1e-4e20-8eea-4b5302d99321 is now active with session: c3cbbad0-a1b8-4f74-9db4-9c8973c6310c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783743561Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783753041Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.78376106Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"session_id":"c3cbbad0-a1b8-4f74-9db4-9c8973c6310c"} {"msg":"Activated sub_volume ab278692-fb1e-4e20-8eea-4b5302d99321","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.78377077Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.783779399Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.784080229Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"task":"crutest"} 01/30 Write block 566 len 24576 data: 12 14 13 15 15 14 02/30 Write block 454 len 36864 data: 9 9 10 13 16 14 14 12 10 03/30 Read block 61 len 16384 04/30 Write block 48 len 8192 data: 12 11 05/30 Read block 98 len 40960 06/30 Write block 473 len 36864 data: 12 9 10 10 10 14 12 11 13 07/30 Write block 552 len 36864 data: 17 18 17 19 16 12 15 16 15 08/30 Read block 156 len 36864 09/30 Write block 228 len 28672 data: 15 11 11 13 13 10 9 10/30 Read block 402 len 36864 11/30 Read block 301 len 20480 12/30 Read block 16 len 16384 13/30 Read block 372 len 24576 14/30 Write block 343 len 28672 data: 15 16 14 10 10 8 15 15/30 Read block 475 len 24576 16/30 Write block 550 len 20480 data: 17 16 18 19 18 17/30 Read block 156 len 24576 18/30 Read block 382 len 8192 19/30 Read block 113 len 32768 20/30 Read block 70 len 40960 21/30 Read block 458 len 16384 22/30 Read block 488 len 20480 23/30 Read block 523 len 4096 24/30 Write block 255 len 8192 data: 12 13 25/30 Write block 385 len 4096 data: 14 26/30 Read block 176 len 32768 27/30 Write block 234 len 12288 data: 10 11 11 28/30 Read block 513 len 40960 29/30 Read block 301 len 4096 30/30 Write block 47 len 36864 data: 13 13 12 12 13 10 9 10 11 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Sent false 1001 Acked Write 9 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Read 10 Done Done Sent false 1005 Acked Write 9 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Read 9 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Write 5 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:52.793093085Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:52.793391265Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1825,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 297 297 297 F F F 1 020-039 163 163 163 289 289 289 F F F 2 040-059 166 166 166 296 296 296 T T F <--- 3 060-079 169 169 169 305 305 305 F F F 4 080-099 168 168 168 301 301 301 F F F 5 100-119 168 168 168 301 301 301 F F F 6 120-139 169 169 169 305 305 305 F F F 7 140-159 169 169 169 305 305 305 F F F 8 160-179 167 167 167 298 298 298 F F F 9 180-199 169 169 169 305 305 305 F F F 10 200-219 169 169 169 305 305 305 F F F 11 220-239 167 167 167 298 298 298 T T F <--- 12 240-259 168 168 168 300 300 300 T T F <--- 13 260-279 168 168 168 300 300 300 F F F 14 280-299 165 165 165 294 294 294 F F F 15 300-319 165 165 165 294 294 294 F F F 16 320-339 169 169 169 305 305 305 F F F 17 340-359 164 164 164 290 290 290 T T F <--- 18 360-379 167 167 167 298 298 298 F F F 19 380-399 169 169 169 305 305 305 T T F <--- 20 400-419 169 169 169 305 305 305 F F F 21 420-439 167 167 167 298 298 298 F F F 22 440-459 166 166 166 295 295 295 T T F <--- 23 460-479 165 165 165 294 294 294 T T F <--- 24 480-499 165 165 165 294 294 294 T T F <--- 25 500-519 164 164 164 291 291 291 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 164 164 164 292 292 292 T T F <--- 28 560-579 164 164 164 292 292 292 T T F <--- 29 580-599 168 168 168 304 304 304 F F F Max gen: 169, Max flush: 305 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 297 297 297 F F F 1 020-039 163 163 163 289 289 289 F F F 2 040-059 170 170 170 306 306 306 F F F 3 060-079 169 169 169 305 305 305 F F F 4 080-099 168 168 168 301 301 301 F F F 5 100-119 168 168 168 301 301 301 F F F 6 120-139 169 169 169 305 305 305 F F F 7 140-159 169 169 169 305 305 305 F F F 8 160-179 167 167 167 298 298 298 F F F 9 180-199 169 169 169 305 305 305 F F F 10 200-219 169 169 169 305 305 305 F F F 11 220-239 170 170 170 306 306 306 F F F 12 240-259 170 170 170 306 306 306 F F F 13 260-279 168 168 168 300 300 300 F F F 14 280-299 165 165 165 294 294 294 F F F 15 300-319 165 165 165 294 294 294 F F F 16 320-339 169 169 169 305 305 305 F F F 17 340-359 170 170 170 306 306 306 F F F 18 360-379 167 167 167 298 298 298 F F F 19 380-399 170 170 170 306 306 306 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 167 167 167 298 298 298 F F F 22 440-459 170 170 170 306 306 306 F F F 23 460-479 170 170 170 306 306 306 F F F 24 480-499 170 170 170 306 306 306 F F F 25 500-519 164 164 164 291 291 291 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 170 170 170 306 306 306 F F F 28 560-579 170 170 170 306 306 306 F F F 29 580-599 168 168 168 304 304 304 F F F Max gen: 170, Max flush: 306 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.034574729Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.042881824Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043172834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043214771Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043224551Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.04323667Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833} {"msg":"Crucible 3a3bcd52-b08a-422f-824d-d9da19016522 has session id: 311a6fb3-1490-40e5-b17d-562659459622","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043496102Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"Upstairs opts: Upstairs UUID: 3a3bcd52-b08a-422f-824d-d9da19016522, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043551918Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"Crucible stats registered with UUID: 3a3bcd52-b08a-422f-824d-d9da19016522","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043563038Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:57.043572507Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043737956Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"io task","client":"1","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043755224Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"io task","client":"0","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043765814Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"io task","client":"2","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043873886Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043908204Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833} {"msg":"3a3bcd52-b08a-422f-824d-d9da19016522 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043947141Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043956711Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"client":"0","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.04396683Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"client":"1","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043976979Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"client":"2","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043987059Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"io task","client":"2","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.043997358Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"io task","client":"1","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.044025416Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"io task","client":"0","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"ds_connection connected from Ok(127.0.0.1:51250)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.044052364Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"io task","client":"2","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"ds_connection connected from Ok(127.0.0.1:35274)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.044065833Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"io task","client":"0","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"ds_connection connected from Ok(127.0.0.1:48728)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.044154847Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"io task","client":"1","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.044726158Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"client":"0","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.044748137Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"client":"2","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.044797544Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"client":"1","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045042417Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045066485Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"[0]R flush_numbers[0..12]: [297, 289, 306, 305, 301, 301, 305, 305, 298, 305, 305, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045089304Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"downstairs","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"[0]R generation[0..12]: [166, 163, 170, 169, 168, 168, 169, 169, 167, 169, 169, 170]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045101103Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"downstairs","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045112512Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"downstairs","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"[1]R flush_numbers[0..12]: [297, 289, 306, 305, 301, 301, 305, 305, 298, 305, 305, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045123651Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"downstairs","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"[1]R generation[0..12]: [166, 163, 170, 169, 168, 168, 169, 169, 167, 169, 169, 170]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045134621Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"downstairs","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"downstairs","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"[2]R flush_numbers[0..12]: [297, 289, 306, 305, 301, 301, 305, 305, 298, 305, 305, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045159509Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"downstairs","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"[2]R generation[0..12]: [166, 163, 170, 169, 168, 168, 169, 169, 167, 169, 169, 170]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045170428Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"downstairs","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045181697Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"downstairs","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"Max found gen is 171","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045191317Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"downstairs","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"Generation requested: 172 >= found:171","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045201536Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"downstairs","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045211155Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"downstairs","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045220895Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"":"downstairs","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045231434Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045240843Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"client":"0","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045251203Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"client":"1","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045261422Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"client":"2","":"downstairs","session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045271931Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045281401Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"3a3bcd52-b08a-422f-824d-d9da19016522 is now active with session: 311a6fb3-1490-40e5-b17d-562659459622","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.04529163Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045302639Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045310719Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"session_id":"311a6fb3-1490-40e5-b17d-562659459622"} {"msg":"Activated sub_volume 3a3bcd52-b08a-422f-824d-d9da19016522","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045320348Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045328827Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.045675894Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"task":"crutest"} 01/30 Write block 544 len 4096 data: 14 02/30 Flush 03/30 Write block 329 len 32768 data: 17 15 14 15 16 14 13 15 04/30 Read block 53 len 32768 05/30 Write block 265 len 12288 data: 10 9 7 06/30 Read block 7 len 24576 07/30 Read block 472 len 28672 08/30 Read block 189 len 12288 09/30 Write block 447 len 36864 data: 7 9 11 10 10 10 9 10 10 10/30 Read block 398 len 20480 11/30 Write block 174 len 16384 data: 14 11 9 7 12/30 Write block 323 len 4096 data: 17 13/30 Write block 592 len 24576 data: 12 11 11 10 9 9 14/30 Read block 167 len 36864 15/30 Flush 16/30 Write block 288 len 40960 data: 9 8 6 7 9 11 12 13 14 17 17/30 Read block 189 len 12288 18/30 Write block 62 len 36864 data: 9 9 9 8 6 6 7 9 7 19/30 Write block 303 len 36864 data: 12 14 15 19 19 16 15 15 19 20/30 Read block 181 len 28672 21/30 Flush 22/30 Write block 291 len 36864 data: 8 10 12 13 14 15 18 19 17 23/30 Read block 388 len 12288 24/30 Read block 113 len 32768 25/30 Flush 26/30 Write block 565 len 32768 data: 16 13 15 14 16 16 15 16 27/30 Read block 596 len 4096 28/30 Write block 196 len 36864 data: 13 12 14 18 19 18 16 14 18 29/30 Write block 490 len 4096 data: 11 30/30 Read block 111 len 8192 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Sent Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 8 Done Sent Done false 1003 Acked Read 8 Done Sent Done false 1004 Acked Write 3 Done Sent Done false 1005 Acked Read 6 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Write 9 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Read 9 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Write 9 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Read 2 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 0 30 Done 30 0 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:44:57.054761876Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:44:57.055072884Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1833,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 297 297 297 F F F 1 020-039 163 163 163 289 289 289 F F F 2 040-059 170 170 170 306 306 306 F F F 3 060-079 172 169 172 309 305 309 F F F <--- 4 080-099 168 168 168 301 301 301 F F F 5 100-119 168 168 168 301 301 301 F F F 6 120-139 169 169 169 305 305 305 F F F 7 140-159 169 169 169 305 305 305 F F F 8 160-179 172 167 172 308 298 308 F F F <--- 9 180-199 169 169 169 305 305 305 T F T <--- 10 200-219 169 169 169 305 305 305 T F T <--- 11 220-239 170 170 170 306 306 306 F F F 12 240-259 170 170 170 306 306 306 F F F 13 260-279 172 168 172 308 300 308 F F F <--- 14 280-299 172 165 172 310 294 310 F F F <--- 15 300-319 172 165 172 309 294 309 F F F <--- 16 320-339 172 169 172 308 305 308 F F F <--- 17 340-359 170 170 170 306 306 306 F F F 18 360-379 167 167 167 298 298 298 F F F 19 380-399 170 170 170 306 306 306 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 167 167 167 298 298 298 F F F 22 440-459 172 170 172 308 306 308 F F F <--- 23 460-479 170 170 170 306 306 306 F F F 24 480-499 170 170 170 306 306 306 T F T <--- 25 500-519 164 164 164 291 291 291 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 172 170 172 307 306 307 F F F <--- 28 560-579 170 170 170 306 306 306 T F T <--- 29 580-599 172 168 172 308 304 308 F F F <--- Max gen: 172, Max flush: 310 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 297 297 297 F F F 1 020-039 163 163 163 289 289 289 F F F 2 040-059 170 170 170 306 306 306 F F F 3 060-079 172 172 172 309 309 309 F F F 4 080-099 168 168 168 301 301 301 F F F 5 100-119 168 168 168 301 301 301 F F F 6 120-139 169 169 169 305 305 305 F F F 7 140-159 169 169 169 305 305 305 F F F 8 160-179 172 172 172 308 308 308 F F F 9 180-199 173 173 173 311 311 311 F F F 10 200-219 173 173 173 311 311 311 F F F 11 220-239 170 170 170 306 306 306 F F F 12 240-259 170 170 170 306 306 306 F F F 13 260-279 172 172 172 308 308 308 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 172 172 172 309 309 309 F F F 16 320-339 172 172 172 308 308 308 F F F 17 340-359 170 170 170 306 306 306 F F F 18 360-379 167 167 167 298 298 298 F F F 19 380-399 170 170 170 306 306 306 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 167 167 167 298 298 298 F F F 22 440-459 172 172 172 308 308 308 F F F 23 460-479 170 170 170 306 306 306 F F F 24 480-499 173 173 173 311 311 311 F F F 25 500-519 164 164 164 291 291 291 F F F 26 520-539 161 161 161 285 285 285 F F F 27 540-559 172 172 172 307 307 307 F F F 28 560-579 173 173 173 311 311 311 F F F 29 580-599 172 172 172 308 308 308 F F F Max gen: 173, Max flush: 311 Begin loop 87 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.298526504Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.306833759Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.3071167Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.307143818Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.307152667Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.307164466Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841} {"msg":"Crucible 14bbd72b-9226-4135-89ac-2558e7e0219f has session id: 0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.30755337Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"Upstairs opts: Upstairs UUID: 14bbd72b-9226-4135-89ac-2558e7e0219f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.307584478Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"Crucible stats registered with UUID: 14bbd72b-9226-4135-89ac-2558e7e0219f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.307596137Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:01.307608176Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.307789964Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"io task","client":"0","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.307822102Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"io task","client":"2","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.307833381Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"io task","client":"1","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.307931764Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.307948963Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841} {"msg":"14bbd72b-9226-4135-89ac-2558e7e0219f active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.307982621Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.30799231Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"client":"0","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.308002709Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"client":"1","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.308012969Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"client":"2","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.308022858Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"io task","client":"2","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.308033317Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"io task","client":"1","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.308043457Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"io task","client":"0","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"ds_connection connected from Ok(127.0.0.1:40426)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.308097093Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"io task","client":"2","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"ds_connection connected from Ok(127.0.0.1:39091)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.308108442Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"io task","client":"0","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"ds_connection connected from Ok(127.0.0.1:52520)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.308181367Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"io task","client":"1","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.308760138Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"client":"0","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.308794416Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"client":"1","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.30887526Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"client":"2","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309073287Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309083336Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"[0]R flush_numbers[0..12]: [297, 289, 306, 309, 301, 301, 305, 305, 308, 311, 311, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.30916601Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"downstairs","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"[0]R generation[0..12]: [166, 163, 170, 172, 168, 168, 169, 169, 172, 173, 173, 170]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309177719Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"downstairs","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309189069Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"downstairs","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"[1]R flush_numbers[0..12]: [297, 289, 306, 309, 301, 301, 305, 305, 308, 311, 311, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309200018Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"downstairs","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"[1]R generation[0..12]: [166, 163, 170, 172, 168, 168, 169, 169, 172, 173, 173, 170]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309210787Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"downstairs","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309221986Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"downstairs","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"[2]R flush_numbers[0..12]: [297, 289, 306, 309, 301, 301, 305, 305, 308, 311, 311, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309232886Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"downstairs","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"[2]R generation[0..12]: [166, 163, 170, 172, 168, 168, 169, 169, 172, 173, 173, 170]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309243745Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"downstairs","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309254984Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"downstairs","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309266353Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"downstairs","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309276443Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"downstairs","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309286092Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"downstairs","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309295751Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"":"downstairs","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309305841Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.30931514Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"client":"0","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309325509Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"client":"1","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309335369Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"client":"2","":"downstairs","session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309345438Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309354987Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"14bbd72b-9226-4135-89ac-2558e7e0219f is now active with session: 0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309365327Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309374816Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309382725Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"session_id":"0f7ab65f-225a-4bad-b1c0-a8da4dd1dbe8"} {"msg":"Activated sub_volume 14bbd72b-9226-4135-89ac-2558e7e0219f","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309392255Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309400674Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.309731462Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"task":"crutest"} 01/30 Flush 02/30 Flush 03/30 Write block 520 len 8192 data: 16 14 04/30 Write block 307 len 12288 data: 20 17 16 05/30 Write block 435 len 36864 data: 12 10 10 12 12 13 13 15 10 06/30 Write block 383 len 40960 data: 14 12 15 13 10 11 11 14 16 15 07/30 Read block 461 len 32768 08/30 Read block 445 len 12288 09/30 Write block 148 len 12288 data: 17 18 19 10/30 Read block 592 len 28672 11/30 Write block 470 len 8192 data: 18 16 12/30 Write block 50 len 32768 data: 13 14 11 10 11 12 12 12 13/30 Write block 431 len 40960 data: 14 14 14 12 13 11 11 13 13 14 14/30 Write block 127 len 36864 data: 14 14 12 11 13 16 11 14 12 15/30 Write block 473 len 4096 data: 13 16/30 Read block 25 len 40960 17/30 Write block 247 len 28672 data: 13 9 9 9 10 8 10 18/30 Read block 282 len 40960 19/30 Write block 150 len 8192 data: 20 20 20/30 Write block 269 len 40960 data: 9 8 8 8 8 7 10 10 9 10 21/30 Write block 130 len 24576 data: 12 14 17 12 15 13 22/30 Flush 23/30 Write block 458 len 36864 data: 17 15 15 13 11 13 14 17 18 24/30 Write block 385 len 4096 data: 16 25/30 Read block 173 len 28672 26/30 Write block 9 len 20480 data: 15 14 14 13 14 27/30 Read block 465 len 4096 28/30 Write block 359 len 40960 data: 15 16 19 14 14 13 15 16 14 15 29/30 Read block 599 len 4096 30/30 Write block 272 len 12288 data: 9 9 8 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 2 Done Done Sent false 1003 Acked Write 3 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Write 10 Done Done Sent false 1006 Acked Read 8 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Write 2 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Read 10 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Write 2 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 6 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 29 31 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:01.31784271Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:01.318149399Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1841,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 297 297 297 T T F <--- 1 020-039 163 163 163 289 289 289 F F F 2 040-059 174 174 170 314 314 306 F F F <--- 3 060-079 172 172 172 309 309 309 F F F 4 080-099 168 168 168 301 301 301 F F F 5 100-119 168 168 168 301 301 301 F F F 6 120-139 174 174 169 314 314 305 F F F <--- 7 140-159 174 174 169 314 314 305 F F F <--- 8 160-179 172 172 172 308 308 308 F F F 9 180-199 173 173 173 311 311 311 F F F 10 200-219 173 173 173 311 311 311 F F F 11 220-239 170 170 170 306 306 306 F F F 12 240-259 174 174 170 314 314 306 F F F <--- 13 260-279 174 174 172 314 314 308 T T F <--- 14 280-299 172 172 172 310 310 310 F F F 15 300-319 174 174 172 314 314 309 F F F <--- 16 320-339 172 172 172 308 308 308 F F F 17 340-359 170 170 170 306 306 306 T T F <--- 18 360-379 167 167 167 298 298 298 T T F <--- 19 380-399 174 174 170 314 314 306 T T F <--- 20 400-419 169 169 169 305 305 305 F F F 21 420-439 174 174 167 314 314 298 F F F <--- 22 440-459 174 174 172 314 314 308 T T F <--- 23 460-479 174 174 170 314 314 306 T T F <--- 24 480-499 173 173 173 311 311 311 F F F 25 500-519 164 164 164 291 291 291 F F F 26 520-539 174 174 161 314 314 285 F F F <--- 27 540-559 172 172 172 307 307 307 F F F 28 560-579 173 173 173 311 311 311 F F F 29 580-599 172 172 172 308 308 308 F F F Max gen: 174, Max flush: 314 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 315 315 315 F F F 1 020-039 163 163 163 289 289 289 F F F 2 040-059 174 174 174 314 314 314 F F F 3 060-079 172 172 172 309 309 309 F F F 4 080-099 168 168 168 301 301 301 F F F 5 100-119 168 168 168 301 301 301 F F F 6 120-139 174 174 174 314 314 314 F F F 7 140-159 174 174 174 314 314 314 F F F 8 160-179 172 172 172 308 308 308 F F F 9 180-199 173 173 173 311 311 311 F F F 10 200-219 173 173 173 311 311 311 F F F 11 220-239 170 170 170 306 306 306 F F F 12 240-259 174 174 174 314 314 314 F F F 13 260-279 175 175 175 315 315 315 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 174 174 174 314 314 314 F F F 16 320-339 172 172 172 308 308 308 F F F 17 340-359 175 175 175 315 315 315 F F F 18 360-379 175 175 175 315 315 315 F F F 19 380-399 175 175 175 315 315 315 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 174 174 174 314 314 314 F F F 22 440-459 175 175 175 315 315 315 F F F 23 460-479 175 175 175 315 315 315 F F F 24 480-499 173 173 173 311 311 311 F F F 25 500-519 164 164 164 291 291 291 F F F 26 520-539 174 174 174 314 314 314 F F F 27 540-559 172 172 172 307 307 307 F F F 28 560-579 173 173 173 311 311 311 F F F 29 580-599 172 172 172 308 308 308 F F F Max gen: 175, Max flush: 315 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.565781854Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:05.572842714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.57290728Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.57981519Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580058643Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580081661Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580090541Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.58010257Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849} {"msg":"Crucible 75d7be45-eff2-44de-9d91-6f218579341c has session id: 18551184-23c3-415e-8325-cc71f4731680","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580471175Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"Upstairs opts: Upstairs UUID: 75d7be45-eff2-44de-9d91-6f218579341c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580501213Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"Crucible stats registered with UUID: 75d7be45-eff2-44de-9d91-6f218579341c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580512832Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:05.580522541Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580682491Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"io task","client":"2","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580713298Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"io task","client":"0","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580724638Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"io task","client":"1","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.58084445Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580869308Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849} {"msg":"75d7be45-eff2-44de-9d91-6f218579341c active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580926934Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580936733Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"client":"0","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580947063Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"client":"1","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580957052Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"client":"2","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.580967211Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"io task","client":"2","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.58097764Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"io task","client":"0","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.58098811Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"io task","client":"1","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"ds_connection connected from Ok(127.0.0.1:43024)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.581034587Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"io task","client":"0","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"ds_connection connected from Ok(127.0.0.1:60167)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.581046036Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"io task","client":"2","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"ds_connection connected from Ok(127.0.0.1:65510)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.581056745Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"io task","client":"1","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.581733099Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"client":"1","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.581749508Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"client":"2","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.581812304Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"client":"0","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582086315Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582131212Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"[0]R flush_numbers[0..12]: [315, 289, 314, 309, 301, 301, 314, 314, 308, 311, 311, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582180319Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"downstairs","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"[0]R generation[0..12]: [175, 163, 174, 172, 168, 168, 174, 174, 172, 173, 173, 170]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582191988Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"downstairs","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582203297Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"downstairs","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"[1]R flush_numbers[0..12]: [315, 289, 314, 309, 301, 301, 314, 314, 308, 311, 311, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582214256Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"downstairs","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"[1]R generation[0..12]: [175, 163, 174, 172, 168, 168, 174, 174, 172, 173, 173, 170]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582225226Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"downstairs","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582236605Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"downstairs","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"[2]R flush_numbers[0..12]: [315, 289, 314, 309, 301, 301, 314, 314, 308, 311, 311, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582247564Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"downstairs","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"[2]R generation[0..12]: [175, 163, 174, 172, 168, 168, 174, 174, 172, 173, 173, 170]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582258463Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"downstairs","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582269673Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"downstairs","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582279472Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"downstairs","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582289481Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"downstairs","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582299261Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"downstairs","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.58231052Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"":"downstairs","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582320619Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582329628Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"client":"0","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582339878Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"client":"1","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582350187Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"client":"2","":"downstairs","session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582360666Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582370236Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"75d7be45-eff2-44de-9d91-6f218579341c is now active with session: 18551184-23c3-415e-8325-cc71f4731680","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582380405Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582389844Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582397804Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"session_id":"18551184-23c3-415e-8325-cc71f4731680"} {"msg":"Activated sub_volume 75d7be45-eff2-44de-9d91-6f218579341c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582407243Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582415763Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.582728321Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"task":"crutest"} 01/30 Write block 484 len 4096 data: 13 02/30 Write block 15 len 40960 data: 9 7 8 10 10 11 12 13 12 12 03/30 Flush 04/30 Write block 349 len 36864 data: 16 17 17 15 14 13 15 14 14 05/30 Read block 208 len 40960 06/30 Write block 57 len 40960 data: 13 12 9 9 10 10 10 10 9 7 07/30 Read block 540 len 20480 08/30 Read block 594 len 8192 09/30 Write block 22 len 12288 data: 14 13 13 10/30 Read block 321 len 16384 11/30 Read block 576 len 32768 12/30 Read block 491 len 32768 13/30 Write block 341 len 24576 data: 14 15 16 17 15 11 14/30 Write block 34 len 28672 data: 14 16 17 14 14 15 15 15/30 Read block 447 len 8192 16/30 Read block 30 len 12288 17/30 Read block 387 len 40960 18/30 Read block 71 len 36864 19/30 Read block 243 len 28672 20/30 Read block 379 len 40960 21/30 Read block 451 len 12288 22/30 Flush 23/30 Write block 249 len 8192 data: 10 10 24/30 Flush 25/30 Write block 569 len 8192 data: 17 17 26/30 Write block 105 len 36864 data: 9 8 12 14 13 13 12 9 9 27/30 Write block 1 len 40960 data: 7 8 8 11 13 13 16 15 16 15 28/30 Read block 91 len 12288 29/30 Read block 272 len 28672 30/30 Write block 516 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 10 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 9 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Write 10 Sent Done Done false 1006 Acked Read 5 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Read 3 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Read 9 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Read 10 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 2 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 1 31 Done 1 29 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:05.591757487Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:05.592068836Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1849,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 176 176 315 316 316 F T T <--- 1 020-039 163 176 176 289 317 317 F F F <--- 2 040-059 174 176 176 314 317 317 F F F <--- 3 060-079 172 176 176 309 317 317 F F F <--- 4 080-099 168 168 168 301 301 301 F F F 5 100-119 168 168 168 301 301 301 F T T <--- 6 120-139 174 174 174 314 314 314 F F F 7 140-159 174 174 174 314 314 314 F F F 8 160-179 172 172 172 308 308 308 F F F 9 180-199 173 173 173 311 311 311 F F F 10 200-219 173 173 173 311 311 311 F F F 11 220-239 170 170 170 306 306 306 F F F 12 240-259 174 176 176 314 318 318 F F F <--- 13 260-279 175 175 175 315 315 315 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 174 174 174 314 314 314 F F F 16 320-339 172 172 172 308 308 308 F F F 17 340-359 175 176 176 315 317 317 F F F <--- 18 360-379 175 175 175 315 315 315 F F F 19 380-399 175 175 175 315 315 315 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 174 174 174 314 314 314 F F F 22 440-459 175 175 175 315 315 315 F F F 23 460-479 175 175 175 315 315 315 F F F 24 480-499 173 176 176 311 316 316 T F F <--- 25 500-519 164 164 164 291 291 291 F T T <--- 26 520-539 174 174 174 314 314 314 F F F 27 540-559 172 172 172 307 307 307 F F F 28 560-579 173 173 173 311 311 311 F T T <--- 29 580-599 172 172 172 308 308 308 F F F Max gen: 176, Max flush: 318 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 177 177 177 319 319 319 F F F 1 020-039 176 176 176 317 317 317 F F F 2 040-059 176 176 176 317 317 317 F F F 3 060-079 176 176 176 317 317 317 F F F 4 080-099 168 168 168 301 301 301 F F F 5 100-119 177 177 177 319 319 319 F F F 6 120-139 174 174 174 314 314 314 F F F 7 140-159 174 174 174 314 314 314 F F F 8 160-179 172 172 172 308 308 308 F F F 9 180-199 173 173 173 311 311 311 F F F 10 200-219 173 173 173 311 311 311 F F F 11 220-239 170 170 170 306 306 306 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 175 175 175 315 315 315 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 174 174 174 314 314 314 F F F 16 320-339 172 172 172 308 308 308 F F F 17 340-359 176 176 176 317 317 317 F F F 18 360-379 175 175 175 315 315 315 F F F 19 380-399 175 175 175 315 315 315 F F F 20 400-419 169 169 169 305 305 305 F F F 21 420-439 174 174 174 314 314 314 F F F 22 440-459 175 175 175 315 315 315 F F F 23 460-479 175 175 175 315 315 315 F F F 24 480-499 176 176 176 316 316 316 F F F 25 500-519 177 177 177 319 319 319 F F F 26 520-539 174 174 174 314 314 314 F F F 27 540-559 172 172 172 307 307 307 F F F 28 560-579 177 177 177 319 319 319 F F F 29 580-599 172 172 172 308 308 308 F F F Max gen: 177, Max flush: 319 Begin loop 89 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.837542398Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.845951196Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.846234337Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.846261565Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.846272784Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.846284673Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857} {"msg":"Crucible b2bdadad-3a5b-440e-8bc2-f5271a2390fa has session id: e2542491-2907-4e80-a6ef-2974fa7b722c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.846594942Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"Upstairs opts: Upstairs UUID: b2bdadad-3a5b-440e-8bc2-f5271a2390fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.846614361Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"Crucible stats registered with UUID: b2bdadad-3a5b-440e-8bc2-f5271a2390fa","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.84662472Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:09.846638819Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.846817837Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"io task","client":"1","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.846850225Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"io task","client":"0","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.846861524Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"io task","client":"2","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.846970017Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.846994425Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857} {"msg":"b2bdadad-3a5b-440e-8bc2-f5271a2390fa active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.847016523Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.847025723Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"client":"0","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.847035762Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"client":"1","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.847045671Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"client":"2","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.847057831Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"io task","client":"2","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.847078809Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"io task","client":"1","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64691)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.847144405Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"io task","client":"2","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.847171193Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"io task","client":"0","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62192)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.847182852Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"io task","client":"1","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36216)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.847250977Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"io task","client":"0","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.847869065Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"client":"0","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.847887874Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"client":"1","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.847915772Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"client":"2","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848203353Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848213482Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"[0]R flush_numbers[0..12]: [319, 317, 317, 317, 301, 319, 314, 314, 308, 311, 311, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848257139Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"downstairs","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"[0]R generation[0..12]: [177, 176, 176, 176, 168, 177, 174, 174, 172, 173, 173, 170]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848268648Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"downstairs","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848280027Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"downstairs","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"[1]R flush_numbers[0..12]: [319, 317, 317, 317, 301, 319, 314, 314, 308, 311, 311, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848293817Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"downstairs","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"[1]R generation[0..12]: [177, 176, 176, 176, 168, 177, 174, 174, 172, 173, 173, 170]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848304866Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"downstairs","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848316005Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"downstairs","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"[2]R flush_numbers[0..12]: [319, 317, 317, 317, 301, 319, 314, 314, 308, 311, 311, 306]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848326924Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"downstairs","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"[2]R generation[0..12]: [177, 176, 176, 176, 168, 177, 174, 174, 172, 173, 173, 170]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848337784Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"downstairs","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848349003Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"downstairs","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848358832Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"downstairs","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848368881Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"downstairs","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848378581Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"downstairs","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.84838829Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"":"downstairs","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848398609Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848407969Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"client":"0","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848418328Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"client":"1","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848428537Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"client":"2","":"downstairs","session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848438867Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848449926Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"b2bdadad-3a5b-440e-8bc2-f5271a2390fa is now active with session: e2542491-2907-4e80-a6ef-2974fa7b722c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848460325Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848469475Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"session_id":"e2542491-2907-4e80-a6ef-2974fa7b722c"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848478954Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857} {"msg":"Activated sub_volume b2bdadad-3a5b-440e-8bc2-f5271a2390fa","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848487403Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848495943Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.848820931Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"task":"crutest"} 01/30 Flush 02/30 Write block 408 len 8192 data: 11 9 03/30 Write block 88 len 4096 data: 9 04/30 Read block 268 len 16384 05/30 Read block 511 len 24576 06/30 Write block 347 len 24576 data: 11 9 17 18 18 16 07/30 Write block 481 len 24576 data: 14 14 14 14 13 12 08/30 Read block 314 len 4096 09/30 Flush 10/30 Write block 134 len 40960 data: 16 14 15 17 19 20 17 17 16 16 11/30 Write block 191 len 36864 data: 14 15 15 17 19 14 13 15 19 12/30 Write block 23 len 24576 data: 14 14 13 13 10 15 13/30 Read block 134 len 40960 14/30 Read block 272 len 8192 15/30 Flush 16/30 Write block 580 len 36864 data: 16 13 12 11 10 12 11 12 9 17/30 Read block 146 len 32768 18/30 Read block 81 len 36864 19/30 Write block 451 len 16384 data: 11 11 10 11 20/30 Write block 504 len 40960 data: 14 17 13 16 14 16 16 14 10 12 21/30 Flush 22/30 Flush 23/30 Write block 129 len 28672 data: 13 13 15 18 13 17 15 24/30 Write block 404 len 12288 data: 11 12 11 25/30 Read block 373 len 8192 26/30 Flush 27/30 Write block 228 len 28672 data: 16 12 12 14 14 11 11 28/30 Read block 542 len 20480 29/30 Write block 12 len 4096 data: 14 30/30 Write block 104 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 2 Done Done Sent false 1002 Acked Write 1 Done Done Sent false 1003 Acked Read 4 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Write 6 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 30 34 Done 28 28 0 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:09.857075919Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:09.857367839Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1857,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 177 177 177 319 319 319 T T F <--- 1 020-039 178 178 176 322 322 317 F F F <--- 2 040-059 176 176 176 317 317 317 F F F 3 060-079 176 176 176 317 317 317 F F F 4 080-099 178 178 168 321 321 301 F F F <--- 5 100-119 177 177 177 319 319 319 T T F <--- 6 120-139 178 178 174 325 325 314 F F F <--- 7 140-159 178 178 174 322 322 314 F F F <--- 8 160-179 172 172 172 308 308 308 F F F 9 180-199 178 178 173 322 322 311 F F F <--- 10 200-219 173 173 173 311 311 311 F F F 11 220-239 170 170 170 306 306 306 T T F <--- 12 240-259 176 176 176 318 318 318 F F F 13 260-279 175 175 175 315 315 315 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 174 174 174 314 314 314 F F F 16 320-339 172 172 172 308 308 308 F F F 17 340-359 178 178 176 321 321 317 F F F <--- 18 360-379 175 175 175 315 315 315 F F F 19 380-399 175 175 175 315 315 315 F F F 20 400-419 178 178 169 325 325 305 F F F <--- 21 420-439 174 174 174 314 314 314 F F F 22 440-459 178 178 175 323 323 315 F F F <--- 23 460-479 175 175 175 315 315 315 F F F 24 480-499 178 178 176 321 321 316 F F F <--- 25 500-519 178 178 177 323 323 319 F F F <--- 26 520-539 174 174 174 314 314 314 F F F 27 540-559 172 172 172 307 307 307 F F F 28 560-579 177 177 177 319 319 319 F F F 29 580-599 178 178 172 323 323 308 F F F <--- Max gen: 178, Max flush: 325 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 326 326 326 F F F 1 020-039 178 178 178 322 322 322 F F F 2 040-059 176 176 176 317 317 317 F F F 3 060-079 176 176 176 317 317 317 F F F 4 080-099 178 178 178 321 321 321 F F F 5 100-119 179 179 179 326 326 326 F F F 6 120-139 178 178 178 325 325 325 F F F 7 140-159 178 178 178 322 322 322 F F F 8 160-179 172 172 172 308 308 308 F F F 9 180-199 178 178 178 322 322 322 F F F 10 200-219 173 173 173 311 311 311 F F F 11 220-239 179 179 179 326 326 326 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 175 175 175 315 315 315 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 174 174 174 314 314 314 F F F 16 320-339 172 172 172 308 308 308 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 175 175 175 315 315 315 F F F 19 380-399 175 175 175 315 315 315 F F F 20 400-419 178 178 178 325 325 325 F F F 21 420-439 174 174 174 314 314 314 F F F 22 440-459 178 178 178 323 323 323 F F F 23 460-479 175 175 175 315 315 315 F F F 24 480-499 178 178 178 321 321 321 F F F 25 500-519 178 178 178 323 323 323 F F F 26 520-539 174 174 174 314 314 314 F F F 27 540-559 172 172 172 307 307 307 F F F 28 560-579 177 177 177 319 319 319 F F F 29 580-599 178 178 178 323 323 323 F F F Max gen: 179, Max flush: 326 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.10374897Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.111929913Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.112222483Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.112248691Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.112257671Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.11226965Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865} {"msg":"Crucible f88aa276-ae89-4472-9c89-4cbd5db16734 has session id: fe2a1b99-f5d8-43f6-abca-34e1024519f2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.112665523Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"Upstairs opts: Upstairs UUID: f88aa276-ae89-4472-9c89-4cbd5db16734, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.112695091Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"Crucible stats registered with UUID: f88aa276-ae89-4472-9c89-4cbd5db16734","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.11270864Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:14.112718589Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.112895247Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"io task","client":"1","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.112950174Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"io task","client":"0","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.112974122Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"io task","client":"2","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113025388Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113050927Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865} {"msg":"f88aa276-ae89-4472-9c89-4cbd5db16734 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113060236Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113069525Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"client":"0","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113079945Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"client":"1","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113090004Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"client":"2","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113099963Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"io task","client":"2","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113110303Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"io task","client":"1","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113158399Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"io task","client":"0","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:47484)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113176178Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"io task","client":"2","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:52877)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113315169Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"io task","client":"1","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:59356)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113339907Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"io task","client":"0","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113826284Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"client":"0","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113847172Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"client":"1","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.113858592Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"client":"2","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114140963Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114152652Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"[0]R flush_numbers[0..12]: [326, 322, 317, 317, 321, 326, 325, 322, 308, 322, 311, 326]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114193989Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"downstairs","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"[0]R generation[0..12]: [179, 178, 176, 176, 178, 179, 178, 178, 172, 178, 173, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114205588Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"downstairs","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114216867Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"downstairs","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"[1]R flush_numbers[0..12]: [326, 322, 317, 317, 321, 326, 325, 322, 308, 322, 311, 326]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114227917Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"downstairs","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"[1]R generation[0..12]: [179, 178, 176, 176, 178, 179, 178, 178, 172, 178, 173, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114238626Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"downstairs","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114249855Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"downstairs","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"[2]R flush_numbers[0..12]: [326, 322, 317, 317, 321, 326, 325, 322, 308, 322, 311, 326]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114260714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"downstairs","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"[2]R generation[0..12]: [179, 178, 176, 176, 178, 179, 178, 178, 172, 178, 173, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114271574Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"downstairs","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114285383Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"downstairs","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114295272Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"downstairs","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114305351Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"downstairs","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114315021Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"downstairs","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.11432482Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"":"downstairs","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114335279Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114344299Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"client":"0","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114354528Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"client":"1","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114364677Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"client":"2","":"downstairs","session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114375077Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114384616Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"f88aa276-ae89-4472-9c89-4cbd5db16734 is now active with session: fe2a1b99-f5d8-43f6-abca-34e1024519f2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114394795Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114409244Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114417224Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"session_id":"fe2a1b99-f5d8-43f6-abca-34e1024519f2"} {"msg":"Activated sub_volume f88aa276-ae89-4472-9c89-4cbd5db16734","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114426743Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114435162Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.114810007Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"task":"crutest"} 01/30 Write block 541 len 20480 data: 14 15 13 15 13 02/30 Read block 17 len 32768 03/30 Read block 472 len 8192 04/30 Write block 117 len 8192 data: 9 11 05/30 Read block 462 len 32768 06/30 Read block 499 len 12288 07/30 Write block 260 len 28672 data: 15 14 11 13 11 11 10 08/30 Read block 457 len 36864 09/30 Write block 270 len 28672 data: 9 9 10 10 9 11 11 10/30 Write block 594 len 4096 data: 12 11/30 Read block 128 len 24576 12/30 Flush 13/30 Write block 46 len 32768 data: 13 14 14 13 14 15 12 11 14/30 Read block 491 len 40960 15/30 Write block 100 len 40960 data: 11 11 10 10 11 10 9 13 15 14 16/30 Read block 173 len 12288 17/30 Read block 195 len 12288 18/30 Write block 532 len 40960 data: 16 15 16 18 15 16 17 18 15 15 19/30 Read block 16 len 8192 20/30 Read block 554 len 24576 21/30 Flush 22/30 Flush 23/30 Write block 498 len 4096 data: 10 24/30 Write block 507 len 8192 data: 17 15 25/30 Read block 498 len 40960 26/30 Read block 297 len 4096 27/30 Write block 43 len 4096 data: 12 28/30 Read block 301 len 12288 29/30 Write block 432 len 28672 data: 15 15 13 14 12 12 14 30/30 Read block 321 len 28672 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Sent false 1001 Acked Read 8 Done Done Sent false 1002 Acked Read 2 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Read 8 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 8 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Read 3 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Read 1 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 30 31 Done 29 30 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:14.123526244Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:14.123843242Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1865,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 326 326 326 F F F 1 020-039 178 178 178 322 322 322 F F F 2 040-059 180 180 176 328 328 317 T T F <--- 3 060-079 176 176 176 317 317 317 F F F 4 080-099 178 178 178 321 321 321 F F F 5 100-119 180 180 179 328 328 326 F F F <--- 6 120-139 178 178 178 325 325 325 F F F 7 140-159 178 178 178 322 322 322 F F F 8 160-179 172 172 172 308 308 308 F F F 9 180-199 178 178 178 322 322 322 F F F 10 200-219 173 173 173 311 311 311 F F F 11 220-239 179 179 179 326 326 326 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 180 180 175 327 327 315 F F F <--- 14 280-299 172 172 172 310 310 310 F F F 15 300-319 174 174 174 314 314 314 F F F 16 320-339 172 172 172 308 308 308 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 175 175 175 315 315 315 F F F 19 380-399 175 175 175 315 315 315 F F F 20 400-419 178 178 178 325 325 325 F F F 21 420-439 174 174 174 314 314 314 T T F <--- 22 440-459 178 178 178 323 323 323 F F F 23 460-479 175 175 175 315 315 315 F F F 24 480-499 178 178 178 321 321 321 T T F <--- 25 500-519 178 178 178 323 323 323 T T F <--- 26 520-539 180 180 174 328 328 314 F F F <--- 27 540-559 180 180 172 328 328 307 F F F <--- 28 560-579 177 177 177 319 319 319 F F F 29 580-599 180 180 178 327 327 323 F F F <--- Max gen: 180, Max flush: 328 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 326 326 326 F F F 1 020-039 178 178 178 322 322 322 F F F 2 040-059 181 181 181 329 329 329 F F F 3 060-079 176 176 176 317 317 317 F F F 4 080-099 178 178 178 321 321 321 F F F 5 100-119 180 180 180 328 328 328 F F F 6 120-139 178 178 178 325 325 325 F F F 7 140-159 178 178 178 322 322 322 F F F 8 160-179 172 172 172 308 308 308 F F F 9 180-199 178 178 178 322 322 322 F F F 10 200-219 173 173 173 311 311 311 F F F 11 220-239 179 179 179 326 326 326 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 180 180 180 327 327 327 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 174 174 174 314 314 314 F F F 16 320-339 172 172 172 308 308 308 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 175 175 175 315 315 315 F F F 19 380-399 175 175 175 315 315 315 F F F 20 400-419 178 178 178 325 325 325 F F F 21 420-439 181 181 181 329 329 329 F F F 22 440-459 178 178 178 323 323 323 F F F 23 460-479 175 175 175 315 315 315 F F F 24 480-499 181 181 181 329 329 329 F F F 25 500-519 181 181 181 329 329 329 F F F 26 520-539 180 180 180 328 328 328 F F F 27 540-559 180 180 180 328 328 328 F F F 28 560-579 177 177 177 319 319 319 F F F 29 580-599 180 180 180 327 327 327 F F F Max gen: 181, Max flush: 329 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.359117379Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:18.366231905Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.366290551Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.372998814Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.373241828Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.373268336Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.373277145Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.373288984Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873} {"msg":"Crucible 0ca9b5b7-0100-4696-880f-2400fef10d04 has session id: 6301faff-e6cf-492f-a7c2-d64ce05b223c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.373616202Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"Upstairs opts: Upstairs UUID: 0ca9b5b7-0100-4696-880f-2400fef10d04, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.37364627Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"Crucible stats registered with UUID: 0ca9b5b7-0100-4696-880f-2400fef10d04","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.373657779Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:18.373667369Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.373820128Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"io task","client":"1","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.373849636Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"io task","client":"0","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.373863095Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"io task","client":"2","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.373964038Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.373981447Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873} {"msg":"0ca9b5b7-0100-4696-880f-2400fef10d04 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.373999886Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.374009035Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"client":"0","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.374019065Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"client":"1","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.374029064Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"client":"2","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.374039053Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"io task","client":"2","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.374049533Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"io task","client":"0","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.374059882Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"io task","client":"1","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47710)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.374106979Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"io task","client":"2","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62064)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.374138917Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"io task","client":"0","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58893)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.37423335Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"io task","client":"1","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.374879716Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"client":"1","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.374904084Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"client":"2","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.374925813Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"client":"0","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375241062Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375284819Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"[0]R flush_numbers[0..12]: [326, 322, 329, 317, 321, 328, 325, 322, 308, 322, 311, 326]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375341485Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"downstairs","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"[0]R generation[0..12]: [179, 178, 181, 176, 178, 180, 178, 178, 172, 178, 173, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375353074Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"downstairs","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375364363Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"downstairs","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"[1]R flush_numbers[0..12]: [326, 322, 329, 317, 321, 328, 325, 322, 308, 322, 311, 326]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375375442Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"downstairs","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"[1]R generation[0..12]: [179, 178, 181, 176, 178, 180, 178, 178, 172, 178, 173, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375386282Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"downstairs","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375397411Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"downstairs","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"[2]R flush_numbers[0..12]: [326, 322, 329, 317, 321, 328, 325, 322, 308, 322, 311, 326]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.37540842Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"downstairs","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"[2]R generation[0..12]: [179, 178, 181, 176, 178, 180, 178, 178, 172, 178, 173, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375419439Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"downstairs","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375430799Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"downstairs","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375440448Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"downstairs","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375450427Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"downstairs","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"Next flush: 330","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375461487Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"downstairs","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375471106Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"":"downstairs","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375481205Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375490245Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"client":"0","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375500194Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"client":"1","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375510193Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"client":"2","":"downstairs","session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375520363Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375529802Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"0ca9b5b7-0100-4696-880f-2400fef10d04 is now active with session: 6301faff-e6cf-492f-a7c2-d64ce05b223c","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375539971Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375549481Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.37555761Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"session_id":"6301faff-e6cf-492f-a7c2-d64ce05b223c"} {"msg":"Activated sub_volume 0ca9b5b7-0100-4696-880f-2400fef10d04","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375567149Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375575759Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.375887628Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"task":"crutest"} 01/30 Write block 386 len 28672 data: 14 11 12 12 15 17 16 02/30 Read block 72 len 40960 03/30 Read block 247 len 20480 04/30 Write block 577 len 40960 data: 16 15 15 17 14 13 12 11 13 12 05/30 Flush 06/30 Write block 548 len 16384 data: 17 18 18 17 07/30 Write block 315 len 8192 data: 19 23 08/30 Write block 198 len 36864 data: 16 20 20 19 17 15 19 16 16 09/30 Write block 585 len 32768 data: 14 13 13 10 9 11 15 13 10/30 Write block 146 len 8192 data: 19 18 11/30 Read block 232 len 12288 12/30 Read block 99 len 28672 13/30 Read block 545 len 24576 14/30 Read block 161 len 32768 15/30 Flush 16/30 Write block 132 len 28672 data: 19 14 18 16 16 18 20 17/30 Write block 92 len 28672 data: 13 16 14 10 11 13 16 18/30 Flush 19/30 Write block 211 len 24576 data: 17 15 13 11 14 15 20/30 Write block 54 len 12288 data: 12 13 13 21/30 Write block 586 len 16384 data: 14 14 11 10 22/30 Read block 571 len 12288 23/30 Flush 24/30 Write block 54 len 20480 data: 13 14 14 14 13 25/30 Read block 393 len 32768 26/30 Read block 404 len 28672 27/30 Write block 50 len 36864 data: 15 16 13 12 14 15 15 15 14 28/30 Read block 382 len 24576 29/30 Read block 405 len 8192 30/30 Read block 389 len 28672 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 4 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Write 9 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Write 2 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Write 7 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 6 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Read 3 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 5 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Write 9 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 21 0 0 21 Done 4 25 25 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:18.384759854Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:18.385062083Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1873,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 326 326 326 F F F 1 020-039 178 178 178 322 322 322 F F F 2 040-059 181 182 182 329 333 333 F T T <--- 3 060-079 176 176 176 317 317 317 F F F 4 080-099 178 182 182 321 332 332 F F F <--- 5 100-119 180 180 180 328 328 328 F F F 6 120-139 178 182 182 325 332 332 F F F <--- 7 140-159 178 182 182 322 331 331 F F F <--- 8 160-179 172 172 172 308 308 308 F F F 9 180-199 178 182 182 322 331 331 T F F <--- 10 200-219 173 182 182 311 333 333 T F F <--- 11 220-239 179 179 179 326 326 326 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 180 180 180 327 327 327 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 174 182 182 314 331 331 T F F <--- 16 320-339 172 172 172 308 308 308 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 175 175 175 315 315 315 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 178 178 178 325 325 325 F F F 21 420-439 181 181 181 329 329 329 F F F 22 440-459 178 178 178 323 323 323 F F F 23 460-479 175 175 175 315 315 315 F F F 24 480-499 181 181 181 329 329 329 F F F 25 500-519 181 181 181 329 329 329 F F F 26 520-539 180 180 180 328 328 328 F F F 27 540-559 180 182 182 328 331 331 T F F <--- 28 560-579 182 182 182 330 330 330 F F F 29 580-599 182 182 182 330 333 333 T F F <--- Max gen: 182, Max flush: 333 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 326 326 326 F F F 1 020-039 178 178 178 322 322 322 F F F 2 040-059 183 183 183 334 334 334 F F F 3 060-079 176 176 176 317 317 317 F F F 4 080-099 182 182 182 332 332 332 F F F 5 100-119 180 180 180 328 328 328 F F F 6 120-139 182 182 182 332 332 332 F F F 7 140-159 182 182 182 331 331 331 F F F 8 160-179 172 172 172 308 308 308 F F F 9 180-199 182 182 182 331 331 331 F F F 10 200-219 182 182 182 333 333 333 F F F 11 220-239 179 179 179 326 326 326 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 180 180 180 327 327 327 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 182 182 182 331 331 331 F F F 16 320-339 172 172 172 308 308 308 F F F 17 340-359 178 178 178 321 321 321 F F F 18 360-379 175 175 175 315 315 315 F F F 19 380-399 182 182 182 330 330 330 F F F 20 400-419 178 178 178 325 325 325 F F F 21 420-439 181 181 181 329 329 329 F F F 22 440-459 178 178 178 323 323 323 F F F 23 460-479 175 175 175 315 315 315 F F F 24 480-499 181 181 181 329 329 329 F F F 25 500-519 181 181 181 329 329 329 F F F 26 520-539 180 180 180 328 328 328 F F F 27 540-559 182 182 182 331 331 331 F F F 28 560-579 182 182 182 330 330 330 F F F 29 580-599 182 182 182 333 333 333 F F F Max gen: 183, Max flush: 334 Begin loop 92 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.624804556Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.633165057Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.633445408Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.633471846Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.633483425Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.633495214Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881} {"msg":"Crucible 9b09a098-c100-4770-ae8c-8da559990ef0 has session id: c0c7014b-701f-4a41-8900-2962f05e0d9e","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.633890687Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"Upstairs opts: Upstairs UUID: 9b09a098-c100-4770-ae8c-8da559990ef0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.633911886Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"Crucible stats registered with UUID: 9b09a098-c100-4770-ae8c-8da559990ef0","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.633922155Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:22.633935744Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.634111672Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"io task","client":"2","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.634127921Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"io task","client":"0","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.63413843Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"io task","client":"1","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.634259182Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.634277781Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881} {"msg":"9b09a098-c100-4770-ae8c-8da559990ef0 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.634311419Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.634321168Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"client":"0","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.634331187Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"client":"1","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.634341277Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"client":"2","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.634353056Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"io task","client":"2","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.634363925Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"io task","client":"0","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.634407922Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"io task","client":"1","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56260)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.63443421Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"io task","client":"2","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49646)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.6344462Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"io task","client":"0","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"ds_connection connected from Ok(127.0.0.1:56304)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.634457389Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"io task","client":"1","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635103135Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"client":"0","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635129693Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"client":"1","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635245015Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"client":"2","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635455031Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.63546512Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"[0]R flush_numbers[0..12]: [326, 322, 334, 317, 332, 328, 332, 331, 308, 331, 333, 326]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635550994Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"downstairs","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"[0]R generation[0..12]: [179, 178, 183, 176, 182, 180, 182, 182, 172, 182, 182, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635562724Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"downstairs","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635574083Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"downstairs","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"[1]R flush_numbers[0..12]: [326, 322, 334, 317, 332, 328, 332, 331, 308, 331, 333, 326]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635587682Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"downstairs","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"[1]R generation[0..12]: [179, 178, 183, 176, 182, 180, 182, 182, 172, 182, 182, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635598701Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"downstairs","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.63561008Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"downstairs","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"[2]R flush_numbers[0..12]: [326, 322, 334, 317, 332, 328, 332, 331, 308, 331, 333, 326]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.63562107Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"downstairs","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"[2]R generation[0..12]: [179, 178, 183, 176, 182, 180, 182, 182, 172, 182, 182, 179]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635631909Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"downstairs","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635643058Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"downstairs","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635652907Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"downstairs","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635662957Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"downstairs","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635672726Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"downstairs","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635682435Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"":"downstairs","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635692585Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635701734Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"client":"0","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635711823Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"client":"1","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635721863Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"client":"2","":"downstairs","session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635731932Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635742541Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"9b09a098-c100-4770-ae8c-8da559990ef0 is now active with session: c0c7014b-701f-4a41-8900-2962f05e0d9e","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635752911Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.6357623Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635770369Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"session_id":"c0c7014b-701f-4a41-8900-2962f05e0d9e"} {"msg":"Activated sub_volume 9b09a098-c100-4770-ae8c-8da559990ef0","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635779759Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.635788228Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.636115866Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"task":"crutest"} 01/30 Write block 439 len 32768 data: 14 15 14 16 11 9 8 6 02/30 Flush 03/30 Flush 04/30 Flush 05/30 Flush 06/30 Write block 227 len 36864 data: 13 17 13 13 15 15 12 12 12 07/30 Write block 523 len 40960 data: 12 14 16 15 14 15 15 14 14 17 08/30 Flush 09/30 Write block 153 len 12288 data: 10 10 12 10/30 Flush 11/30 Write block 395 len 28672 data: 16 16 18 19 20 18 17 12/30 Read block 505 len 4096 13/30 Write block 164 len 36864 data: 15 16 14 17 14 12 14 15 12 14/30 Read block 221 len 24576 15/30 Read block 131 len 4096 16/30 Write block 17 len 24576 data: 9 11 11 12 13 15 17/30 Flush 18/30 Write block 520 len 20480 data: 17 15 13 13 15 19/30 Read block 434 len 16384 20/30 Read block 108 len 36864 21/30 Read block 385 len 8192 22/30 Flush 23/30 Write block 533 len 4096 data: 16 24/30 Read block 445 len 8192 25/30 Flush 26/30 Write block 357 len 36864 data: 15 12 16 17 20 15 15 14 16 27/30 Read block 107 len 24576 28/30 Write block 68 len 36864 data: 8 10 8 7 10 11 11 12 12 29/30 Read block 480 len 28672 30/30 Read block 289 len 4096 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 7 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 6 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 5 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 9 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 1 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 15 15 Done 20 20 5 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) Some(JobId(1009)) Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:22.645515636Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:22.645808436Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1881,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 179 341 341 326 F F F <--- 1 020-039 184 184 178 341 341 322 F F F <--- 2 040-059 183 183 183 334 334 334 F F F 3 060-079 176 176 176 317 317 317 T T F <--- 4 080-099 182 182 182 332 332 332 F F F 5 100-119 180 180 180 328 328 328 F F F 6 120-139 182 182 182 332 332 332 F F F 7 140-159 184 184 184 340 340 340 F F F 8 160-179 184 184 172 341 341 308 F F T <--- 9 180-199 182 182 182 331 331 331 F F F 10 200-219 182 182 182 333 333 333 F F F 11 220-239 184 184 184 339 339 339 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 180 180 180 327 327 327 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 182 182 182 331 331 331 F F F 16 320-339 172 172 172 308 308 308 F F F 17 340-359 178 178 178 321 321 321 T T F <--- 18 360-379 175 175 175 315 315 315 T T F <--- 19 380-399 184 184 182 341 341 330 F F T <--- 20 400-419 184 184 178 341 341 325 F F T <--- 21 420-439 184 184 184 335 335 335 F F F 22 440-459 184 184 184 335 335 335 F F F 23 460-479 175 175 175 315 315 315 F F F 24 480-499 181 181 181 329 329 329 F F F 25 500-519 181 181 181 329 329 329 F F F 26 520-539 184 184 184 343 343 339 F F F <--- 27 540-559 182 182 182 331 331 331 F F F 28 560-579 182 182 182 330 330 330 F F F 29 580-599 182 182 182 333 333 333 F F F Max gen: 184, Max flush: 343 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 184 184 184 341 341 341 F F F 1 020-039 184 184 184 341 341 341 F F F 2 040-059 183 183 183 334 334 334 F F F 3 060-079 185 185 185 344 344 344 F F F 4 080-099 182 182 182 332 332 332 F F F 5 100-119 180 180 180 328 328 328 F F F 6 120-139 182 182 182 332 332 332 F F F 7 140-159 184 184 184 340 340 340 F F F 8 160-179 184 184 184 341 341 341 F F F 9 180-199 182 182 182 331 331 331 F F F 10 200-219 182 182 182 333 333 333 F F F 11 220-239 184 184 184 339 339 339 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 180 180 180 327 327 327 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 182 182 182 331 331 331 F F F 16 320-339 172 172 172 308 308 308 F F F 17 340-359 185 185 185 344 344 344 F F F 18 360-379 185 185 185 344 344 344 F F F 19 380-399 184 184 184 341 341 341 F F F 20 400-419 184 184 184 341 341 341 F F F 21 420-439 184 184 184 335 335 335 F F F 22 440-459 184 184 184 335 335 335 F F F 23 460-479 175 175 175 315 315 315 F F F 24 480-499 181 181 181 329 329 329 F F F 25 500-519 181 181 181 329 329 329 F F F 26 520-539 184 184 184 343 343 343 F F F 27 540-559 182 182 182 331 331 331 F F F 28 560-579 182 182 182 330 330 330 F F F 29 580-599 182 182 182 333 333 333 F F F Max gen: 185, Max flush: 344 Begin loop 93 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.881630165Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.88979813Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890077551Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890104749Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890113728Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890125467Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889} {"msg":"Crucible ae8bd9e6-49bf-4d37-aff0-5f8a35b46bec has session id: c6e91609-dc53-4ddf-ac33-793ac388bf01","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890461714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"Upstairs opts: Upstairs UUID: ae8bd9e6-49bf-4d37-aff0-5f8a35b46bec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890491782Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"Crucible stats registered with UUID: ae8bd9e6-49bf-4d37-aff0-5f8a35b46bec","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890503152Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:26.890512711Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890684599Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"io task","client":"2","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890701498Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"io task","client":"0","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890711857Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"io task","client":"1","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890876516Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890890975Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889} {"msg":"ae8bd9e6-49bf-4d37-aff0-5f8a35b46bec active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890924333Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890933852Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"client":"0","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890943892Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"client":"1","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890953861Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"client":"2","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.89096371Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"io task","client":"2","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.8909742Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"io task","client":"1","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.890984839Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"io task","client":"0","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"ds_connection connected from Ok(127.0.0.1:63734)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.891040365Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"io task","client":"1","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"ds_connection connected from Ok(127.0.0.1:37121)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.891066473Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"io task","client":"0","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"ds_connection connected from Ok(127.0.0.1:62074)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.891078172Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"io task","client":"2","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.891709799Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"client":"2","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.891759956Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"client":"0","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.891779515Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"client":"1","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892021818Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892032178Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"[0]R flush_numbers[0..12]: [341, 341, 334, 344, 332, 328, 332, 340, 341, 331, 333, 339]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892071895Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"downstairs","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"[0]R generation[0..12]: [184, 184, 183, 185, 182, 180, 182, 184, 184, 182, 182, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892083364Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"downstairs","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892094583Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"downstairs","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"[1]R flush_numbers[0..12]: [341, 341, 334, 344, 332, 328, 332, 340, 341, 331, 333, 339]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892105473Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"downstairs","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"[1]R generation[0..12]: [184, 184, 183, 185, 182, 180, 182, 184, 184, 182, 182, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892116322Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"downstairs","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892127531Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"downstairs","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"[2]R flush_numbers[0..12]: [341, 341, 334, 344, 332, 328, 332, 340, 341, 331, 333, 339]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.89213834Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"downstairs","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"[2]R generation[0..12]: [184, 184, 183, 185, 182, 180, 182, 184, 184, 182, 182, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.89214943Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"downstairs","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892160559Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"downstairs","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892170388Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"downstairs","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892183397Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"downstairs","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892193117Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"downstairs","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892202696Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"":"downstairs","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892212745Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892221945Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"client":"0","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892231864Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"client":"1","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892241723Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"client":"2","":"downstairs","session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892251963Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892261272Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"ae8bd9e6-49bf-4d37-aff0-5f8a35b46bec is now active with session: c6e91609-dc53-4ddf-ac33-793ac388bf01","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892271561Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892281031Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.89228896Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"session_id":"c6e91609-dc53-4ddf-ac33-793ac388bf01"} {"msg":"Activated sub_volume ae8bd9e6-49bf-4d37-aff0-5f8a35b46bec","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892298739Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892307299Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.892638446Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"task":"crutest"} 01/30 Write block 330 len 36864 data: 16 15 16 17 15 14 16 12 13 02/30 Write block 403 len 28672 data: 15 12 13 12 11 12 10 03/30 Write block 131 len 36864 data: 16 20 15 19 17 17 19 21 21 04/30 Read block 32 len 4096 05/30 Read block 139 len 16384 06/30 Read block 250 len 28672 07/30 Write block 382 len 36864 data: 15 15 13 17 15 12 13 13 16 08/30 Write block 452 len 16384 data: 12 11 12 11 09/30 Write block 301 len 20480 data: 13 12 13 15 16 10/30 Read block 558 len 24576 11/30 Read block 399 len 20480 12/30 Flush 13/30 Write block 498 len 40960 data: 11 13 14 12 12 12 15 18 14 18 14/30 Flush 15/30 Write block 314 len 36864 data: 20 20 24 25 27 24 19 17 17 16/30 Read block 51 len 8192 17/30 Read block 276 len 28672 18/30 Write block 8 len 24576 data: 16 17 16 15 15 15 19/30 Read block 283 len 36864 20/30 Write block 170 len 32768 data: 15 16 13 15 15 12 10 8 21/30 Read block 367 len 40960 22/30 Read block 278 len 28672 23/30 Write block 103 len 12288 data: 11 12 11 24/30 Flush 25/30 Flush 26/30 Write block 152 len 16384 data: 14 11 11 13 27/30 Read block 326 len 28672 28/30 Write block 493 len 24576 data: 13 13 12 12 11 12 29/30 Read block 137 len 36864 30/30 Read block 544 len 40960 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 4 Done Done Done false 1005 Acked Read 7 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Write 4 Done Sent Done false 1008 Acked Write 5 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Flush 0 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Write 6 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Write 8 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Write 3 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Read 7 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 23 0 24 Done 29 7 30 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:26.902153689Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:26.902445869Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1889,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 184 186 347 341 347 F F F <--- 1 020-039 184 184 184 341 341 341 F F F 2 040-059 183 183 183 334 334 334 F F F 3 060-079 185 185 185 344 344 344 F F F 4 080-099 182 182 182 332 332 332 F F F 5 100-119 186 180 186 347 328 347 F F F <--- 6 120-139 186 182 186 345 332 345 F T F <--- 7 140-159 184 184 184 340 340 340 T F T <--- 8 160-179 186 184 186 347 341 347 F F F <--- 9 180-199 182 182 182 331 331 331 F F F 10 200-219 182 182 182 333 333 333 F F F 11 220-239 184 184 184 339 339 339 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 180 180 180 327 327 327 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 186 182 186 347 331 347 F F F <--- 16 320-339 186 172 186 347 308 347 F T F <--- 17 340-359 185 185 185 344 344 344 F F F 18 360-379 185 185 185 344 344 344 F F F 19 380-399 186 184 186 345 341 345 F T F <--- 20 400-419 186 184 186 345 341 345 F T F <--- 21 420-439 184 184 184 335 335 335 F F F 22 440-459 186 184 186 345 335 345 F F F <--- 23 460-479 175 175 175 315 315 315 F F F 24 480-499 186 181 186 346 329 346 T F T <--- 25 500-519 186 181 186 346 329 346 F F F <--- 26 520-539 184 184 184 343 343 343 F F F 27 540-559 182 182 182 331 331 331 F F F 28 560-579 182 182 182 330 330 330 F F F 29 580-599 182 182 182 333 333 333 F F F Max gen: 186, Max flush: 347 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 347 347 347 F F F 1 020-039 184 184 184 341 341 341 F F F 2 040-059 183 183 183 334 334 334 F F F 3 060-079 185 185 185 344 344 344 F F F 4 080-099 182 182 182 332 332 332 F F F 5 100-119 186 186 186 347 347 347 F F F 6 120-139 186 186 186 345 345 345 F F F 7 140-159 187 187 187 348 348 348 F F F 8 160-179 186 186 186 347 347 347 F F F 9 180-199 182 182 182 331 331 331 F F F 10 200-219 182 182 182 333 333 333 F F F 11 220-239 184 184 184 339 339 339 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 180 180 180 327 327 327 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 186 186 186 347 347 347 F F F 16 320-339 186 186 186 347 347 347 F F F 17 340-359 185 185 185 344 344 344 F F F 18 360-379 185 185 185 344 344 344 F F F 19 380-399 186 186 186 345 345 345 F F F 20 400-419 186 186 186 345 345 345 F F F 21 420-439 184 184 184 335 335 335 F F F 22 440-459 186 186 186 345 345 345 F F F 23 460-479 175 175 175 315 315 315 F F F 24 480-499 187 187 187 348 348 348 F F F 25 500-519 186 186 186 346 346 346 F F F 26 520-539 184 184 184 343 343 343 F F F 27 540-559 182 182 182 331 331 331 F F F 28 560-579 182 182 182 330 330 330 F F F 29 580-599 182 182 182 333 333 333 F F F Max gen: 187, Max flush: 348 Begin loop 94 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.143261048Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.151371756Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.151655037Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.151680985Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.151689855Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.151701614Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897} {"msg":"Crucible 7c5d3d41-258a-4bcc-89f8-fbc120474b99 has session id: eceb3d9b-79bf-47d2-be10-be942402dae7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.15204596Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"Upstairs opts: Upstairs UUID: 7c5d3d41-258a-4bcc-89f8-fbc120474b99, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152079618Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"Crucible stats registered with UUID: 7c5d3d41-258a-4bcc-89f8-fbc120474b99","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152091107Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:31.152100907Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152268445Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"io task","client":"2","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152287654Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"io task","client":"0","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152298203Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"io task","client":"1","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152438794Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152457742Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897} {"msg":"7c5d3d41-258a-4bcc-89f8-fbc120474b99 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152527068Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152538747Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"client":"0","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152548836Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"client":"1","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152558756Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"client":"2","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152568795Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"io task","client":"2","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152579154Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"io task","client":"0","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152589493Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"io task","client":"1","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"ds_connection connected from Ok(127.0.0.1:63494)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.15264163Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"io task","client":"1","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"ds_connection connected from Ok(127.0.0.1:47007)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152654629Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"io task","client":"2","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"ds_connection connected from Ok(127.0.0.1:34371)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.152688027Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"io task","client":"0","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153354371Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"client":"2","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.15337635Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"client":"1","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153470854Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"client":"0","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.15366713Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153677619Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"[0]R flush_numbers[0..12]: [347, 341, 334, 344, 332, 347, 345, 348, 347, 331, 333, 339]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153784032Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"downstairs","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"[0]R generation[0..12]: [186, 184, 183, 185, 182, 186, 186, 187, 186, 182, 182, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153797771Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"downstairs","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153809241Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"downstairs","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"[1]R flush_numbers[0..12]: [347, 341, 334, 344, 332, 347, 345, 348, 347, 331, 333, 339]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.15382027Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"downstairs","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"[1]R generation[0..12]: [186, 184, 183, 185, 182, 186, 186, 187, 186, 182, 182, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153831159Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"downstairs","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153842298Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"downstairs","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"[2]R flush_numbers[0..12]: [347, 341, 334, 344, 332, 347, 345, 348, 347, 331, 333, 339]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153853188Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"downstairs","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"[2]R generation[0..12]: [186, 184, 183, 185, 182, 186, 186, 187, 186, 182, 182, 184]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153863917Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"downstairs","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153875066Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"downstairs","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153884805Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"downstairs","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153894985Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"downstairs","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"Next flush: 349","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153904684Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"downstairs","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153914473Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"":"downstairs","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153924353Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153933592Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"client":"0","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153943621Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"client":"1","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153954871Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"client":"2","":"downstairs","session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.15396493Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153974429Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"7c5d3d41-258a-4bcc-89f8-fbc120474b99 is now active with session: eceb3d9b-79bf-47d2-be10-be942402dae7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153984809Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.153993858Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"session_id":"eceb3d9b-79bf-47d2-be10-be942402dae7"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.154003247Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897} {"msg":"Activated sub_volume 7c5d3d41-258a-4bcc-89f8-fbc120474b99","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.154011697Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.154020096Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.154349924Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"task":"crutest"} 01/30 Write block 556 len 40960 data: 17 13 16 17 16 16 15 17 16 17 02/30 Read block 57 len 8192 03/30 Flush 04/30 Write block 499 len 24576 data: 14 15 13 13 13 16 05/30 Write block 156 len 4096 data: 15 06/30 Write block 234 len 12288 data: 13 13 12 07/30 Read block 379 len 12288 08/30 Read block 578 len 32768 09/30 Flush 10/30 Write block 181 len 20480 data: 8 10 12 10 13 11/30 Write block 233 len 12288 data: 13 14 14 12/30 Read block 177 len 32768 13/30 Read block 426 len 32768 14/30 Flush 15/30 Write block 351 len 40960 data: 19 17 15 14 16 15 16 13 17 18 16/30 Write block 101 len 12288 data: 12 11 12 17/30 Write block 207 len 40960 data: 13 18 15 16 18 16 14 12 15 16 18/30 Write block 426 len 28672 data: 11 12 14 12 12 15 16 19/30 Flush 20/30 Write block 555 len 8192 data: 20 18 21/30 Read block 457 len 40960 22/30 Read block 175 len 24576 23/30 Write block 139 len 16384 data: 22 18 18 17 24/30 Read block 73 len 16384 25/30 Read block 219 len 36864 26/30 Read block 53 len 40960 27/30 Read block 472 len 24576 28/30 Read block 25 len 28672 29/30 Read block 95 len 4096 30/30 Read block 435 len 4096 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 5 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Read 8 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Read 6 Done Sent Done false 1022 Acked Write 4 Done Sent Done false 1023 Acked Read 4 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Read 6 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 1 Done Sent Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 21 0 21 Done 21 0 21 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) Some(JobId(1008)) Some(JobId(1018)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:31.163175083Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:31.163461454Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1897,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 347 347 347 F F F 1 020-039 184 184 184 341 341 341 F F F 2 040-059 183 183 183 334 334 334 F F F 3 060-079 185 185 185 344 344 344 F F F 4 080-099 182 182 182 332 332 332 F F F 5 100-119 188 186 188 352 347 352 F F F <--- 6 120-139 186 186 186 345 345 345 T F T <--- 7 140-159 188 188 188 350 350 350 T F T <--- 8 160-179 186 186 186 347 347 347 F F F 9 180-199 188 182 188 351 331 351 F F F <--- 10 200-219 188 182 188 352 333 352 F F F <--- 11 220-239 188 188 188 351 350 351 F F F <--- 12 240-259 176 176 176 318 318 318 F F F 13 260-279 180 180 180 327 327 327 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 186 186 186 347 347 347 F F F 16 320-339 186 186 186 347 347 347 F F F 17 340-359 188 185 188 352 344 352 F F F <--- 18 360-379 188 185 188 352 344 352 F F F <--- 19 380-399 186 186 186 345 345 345 F F F 20 400-419 186 186 186 345 345 345 F F F 21 420-439 188 184 188 352 335 352 F F F <--- 22 440-459 186 186 186 345 345 345 F F F 23 460-479 175 175 175 315 315 315 F F F 24 480-499 188 188 188 350 350 350 F F F 25 500-519 188 188 188 350 350 350 F F F 26 520-539 184 184 184 343 343 343 F F F 27 540-559 188 188 188 349 349 349 T F T <--- 28 560-579 188 188 188 349 349 349 F F F 29 580-599 182 182 182 333 333 333 F F F Max gen: 188, Max flush: 352 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 347 347 347 F F F 1 020-039 184 184 184 341 341 341 F F F 2 040-059 183 183 183 334 334 334 F F F 3 060-079 185 185 185 344 344 344 F F F 4 080-099 182 182 182 332 332 332 F F F 5 100-119 188 188 188 352 352 352 F F F 6 120-139 189 189 189 353 353 353 F F F 7 140-159 189 189 189 353 353 353 F F F 8 160-179 186 186 186 347 347 347 F F F 9 180-199 188 188 188 351 351 351 F F F 10 200-219 188 188 188 352 352 352 F F F 11 220-239 188 188 188 351 351 351 F F F 12 240-259 176 176 176 318 318 318 F F F 13 260-279 180 180 180 327 327 327 F F F 14 280-299 172 172 172 310 310 310 F F F 15 300-319 186 186 186 347 347 347 F F F 16 320-339 186 186 186 347 347 347 F F F 17 340-359 188 188 188 352 352 352 F F F 18 360-379 188 188 188 352 352 352 F F F 19 380-399 186 186 186 345 345 345 F F F 20 400-419 186 186 186 345 345 345 F F F 21 420-439 188 188 188 352 352 352 F F F 22 440-459 186 186 186 345 345 345 F F F 23 460-479 175 175 175 315 315 315 F F F 24 480-499 188 188 188 350 350 350 F F F 25 500-519 188 188 188 350 350 350 F F F 26 520-539 184 184 184 343 343 343 F F F 27 540-559 189 189 189 353 353 353 F F F 28 560-579 188 188 188 349 349 349 F F F 29 580-599 182 182 182 333 333 333 F F F Max gen: 189, Max flush: 353 Begin loop 95 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.400327932Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:35.407485915Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.4075491Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.414545394Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.414790218Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.414813836Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.414822865Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.414834805Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905} {"msg":"Crucible 19fc343b-ef14-4f86-8c2d-aa9066d9b2d2 has session id: b794752a-ac1a-4469-a3ec-c510e3dc3853","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415162752Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"Upstairs opts: Upstairs UUID: 19fc343b-ef14-4f86-8c2d-aa9066d9b2d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.41519215Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"Crucible stats registered with UUID: 19fc343b-ef14-4f86-8c2d-aa9066d9b2d2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.41520345Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:35.415213129Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415369398Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"io task","client":"1","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415386947Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"io task","client":"0","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415397536Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"io task","client":"2","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415538637Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415552566Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905} {"msg":"19fc343b-ef14-4f86-8c2d-aa9066d9b2d2 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415576564Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415586463Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"client":"0","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415596493Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"client":"1","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415606562Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"client":"2","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415616631Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"io task","client":"2","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415626901Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"io task","client":"0","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.41563728Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"io task","client":"1","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"ds_connection connected from Ok(127.0.0.1:48007)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415677027Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"io task","client":"2","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"ds_connection connected from Ok(127.0.0.1:44506)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415688677Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"io task","client":"1","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"ds_connection connected from Ok(127.0.0.1:47739)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.415747673Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"io task","client":"0","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416290076Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"client":"1","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416348772Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"client":"2","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416466034Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"client":"0","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416596415Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416620243Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"[0]R flush_numbers[0..12]: [347, 341, 334, 344, 332, 352, 353, 353, 347, 351, 352, 351]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416829449Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"downstairs","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"[0]R generation[0..12]: [186, 184, 183, 185, 182, 188, 189, 189, 186, 188, 188, 188]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416841658Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"downstairs","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416858547Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"downstairs","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"[1]R flush_numbers[0..12]: [347, 341, 334, 344, 332, 352, 353, 353, 347, 351, 352, 351]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416869516Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"downstairs","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"[1]R generation[0..12]: [186, 184, 183, 185, 182, 188, 189, 189, 186, 188, 188, 188]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416880535Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"downstairs","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416891805Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"downstairs","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"[2]R flush_numbers[0..12]: [347, 341, 334, 344, 332, 352, 353, 353, 347, 351, 352, 351]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416905174Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"downstairs","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"[2]R generation[0..12]: [186, 184, 183, 185, 182, 188, 189, 189, 186, 188, 188, 188]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416916213Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"downstairs","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416927452Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"downstairs","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416937192Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"downstairs","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416947161Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"downstairs","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"Next flush: 354","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.41695688Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"downstairs","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416967Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"":"downstairs","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416977339Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416986538Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"client":"0","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.416996628Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"client":"1","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.417007057Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"client":"2","":"downstairs","session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.417017406Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.417026795Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"19fc343b-ef14-4f86-8c2d-aa9066d9b2d2 is now active with session: b794752a-ac1a-4469-a3ec-c510e3dc3853","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.417037075Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.417046644Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.417054704Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"session_id":"b794752a-ac1a-4469-a3ec-c510e3dc3853"} {"msg":"Activated sub_volume 19fc343b-ef14-4f86-8c2d-aa9066d9b2d2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.417065463Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.417077652Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"task":"crutest"} Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.417348314Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"task":"crutest"} 01/30 Write block 190 len 24576 data: 14 15 16 16 18 20 02/30 Read block 349 len 12288 03/30 Flush 04/30 Write block 326 len 4096 data: 11 05/30 Write block 111 len 4096 data: 13 06/30 Flush 07/30 Write block 128 len 8192 data: 15 14 08/30 Read block 215 len 32768 09/30 Read block 447 len 24576 10/30 Read block 136 len 8192 11/30 Write block 140 len 28672 data: 19 19 18 17 19 18 20 12/30 Write block 125 len 24576 data: 15 18 15 16 15 14 13/30 Write block 129 len 20480 data: 16 15 17 21 16 14/30 Write block 483 len 36864 data: 15 15 14 13 15 14 13 12 12 15/30 Write block 214 len 12288 data: 13 16 17 16/30 Read block 93 len 24576 17/30 Write block 479 len 40960 data: 13 12 15 15 16 16 15 14 16 15 18/30 Write block 123 len 24576 data: 8 11 16 19 16 17 19/30 Read block 195 len 40960 20/30 Write block 274 len 32768 data: 10 12 12 10 11 9 9 9 21/30 Write block 68 len 8192 data: 9 11 22/30 Write block 176 len 20480 data: 11 9 7 9 7 23/30 Flush 24/30 Write block 502 len 36864 data: 14 14 17 19 15 19 16 17 17 25/30 Read block 211 len 12288 26/30 Read block 534 len 40960 27/30 Read block 455 len 24576 28/30 Write block 239 len 32768 data: 13 16 19 20 19 17 14 17 29/30 Write block 583 len 28672 data: 13 12 15 15 15 12 11 30/30 Read block 57 len 20480 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 1 Sent Done Done false 1004 Acked Write 1 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 2 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Write 7 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Read 3 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Write 8 Sent Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 0 28 Done 0 26 27 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:35.425492689Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:35.425801328Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1905,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 347 347 347 F F F 1 020-039 184 184 184 341 341 341 F F F 2 040-059 183 183 183 334 334 334 F F F 3 060-079 185 190 190 344 356 356 F F F <--- 4 080-099 182 182 182 332 332 332 F F F 5 100-119 188 190 190 352 355 355 F F F <--- 6 120-139 189 190 190 353 356 356 F F F <--- 7 140-159 189 190 190 353 356 356 F F F <--- 8 160-179 186 190 190 347 356 356 F F F <--- 9 180-199 190 190 190 354 356 356 F F F <--- 10 200-219 188 190 190 352 356 356 F F F <--- 11 220-239 188 188 188 351 351 351 F T T <--- 12 240-259 176 176 176 318 318 318 F T T <--- 13 260-279 180 190 190 327 356 356 F F F <--- 14 280-299 172 190 190 310 356 356 F F F <--- 15 300-319 186 186 186 347 347 347 F F F 16 320-339 186 190 190 347 355 355 F F F <--- 17 340-359 188 188 188 352 352 352 F F F 18 360-379 188 188 188 352 352 352 F F F 19 380-399 186 186 186 345 345 345 F F F 20 400-419 186 186 186 345 345 345 F F F 21 420-439 188 188 188 352 352 352 F F F 22 440-459 186 186 186 345 345 345 F F F 23 460-479 175 190 190 315 356 356 F F F <--- 24 480-499 188 190 190 350 356 356 F F F <--- 25 500-519 188 188 188 350 350 350 F T T <--- 26 520-539 184 184 184 343 343 343 F F F 27 540-559 189 189 189 353 353 353 F F F 28 560-579 188 188 188 349 349 349 F F F 29 580-599 182 182 182 333 333 333 F T T <--- Max gen: 190, Max flush: 356 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 347 347 347 F F F 1 020-039 184 184 184 341 341 341 F F F 2 040-059 183 183 183 334 334 334 F F F 3 060-079 190 190 190 356 356 356 F F F 4 080-099 182 182 182 332 332 332 F F F 5 100-119 190 190 190 355 355 355 F F F 6 120-139 190 190 190 356 356 356 F F F 7 140-159 190 190 190 356 356 356 F F F 8 160-179 190 190 190 356 356 356 F F F 9 180-199 190 190 190 356 356 356 F F F 10 200-219 190 190 190 356 356 356 F F F 11 220-239 191 191 191 357 357 357 F F F 12 240-259 191 191 191 357 357 357 F F F 13 260-279 190 190 190 356 356 356 F F F 14 280-299 190 190 190 356 356 356 F F F 15 300-319 186 186 186 347 347 347 F F F 16 320-339 190 190 190 355 355 355 F F F 17 340-359 188 188 188 352 352 352 F F F 18 360-379 188 188 188 352 352 352 F F F 19 380-399 186 186 186 345 345 345 F F F 20 400-419 186 186 186 345 345 345 F F F 21 420-439 188 188 188 352 352 352 F F F 22 440-459 186 186 186 345 345 345 F F F 23 460-479 190 190 190 356 356 356 F F F 24 480-499 190 190 190 356 356 356 F F F 25 500-519 191 191 191 357 357 357 F F F 26 520-539 184 184 184 343 343 343 F F F 27 540-559 189 189 189 353 353 353 F F F 28 560-579 188 188 188 349 349 349 F F F 29 580-599 191 191 191 357 357 357 F F F Max gen: 191, Max flush: 357 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.679531299Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:39.686695332Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.686761127Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.693600692Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.693846545Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.693870593Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.693879693Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.693891572Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913} {"msg":"Crucible 7617f0b6-9878-4142-a70b-50b2dbc5adc3 has session id: 4c201f6a-74cc-4001-b49d-d167be9d6e98","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694241678Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"Upstairs opts: Upstairs UUID: 7617f0b6-9878-4142-a70b-50b2dbc5adc3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694271526Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"Crucible stats registered with UUID: 7617f0b6-9878-4142-a70b-50b2dbc5adc3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694282705Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:39.694294135Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694462293Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"io task","client":"0","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694495401Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"io task","client":"1","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.69450664Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"io task","client":"2","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694595524Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694608583Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913} {"msg":"7617f0b6-9878-4142-a70b-50b2dbc5adc3 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694626142Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694635481Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"client":"0","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694645621Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"client":"1","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.69465573Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"client":"2","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694665849Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"io task","client":"2","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694683788Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"io task","client":"1","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694694677Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"io task","client":"0","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"ds_connection connected from Ok(127.0.0.1:46526)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694743604Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"io task","client":"2","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"ds_connection connected from Ok(127.0.0.1:35141)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694889404Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"io task","client":"1","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"ds_connection connected from Ok(127.0.0.1:52751)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.694913142Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"io task","client":"0","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.69538251Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"client":"1","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695399109Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"client":"2","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695506372Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"client":"0","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695761215Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695771174Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"[0]R flush_numbers[0..12]: [347, 341, 334, 356, 332, 355, 356, 356, 356, 356, 356, 357]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695811551Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"downstairs","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"[0]R generation[0..12]: [186, 184, 183, 190, 182, 190, 190, 190, 190, 190, 190, 191]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695823051Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"downstairs","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.69583444Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"downstairs","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"[1]R flush_numbers[0..12]: [347, 341, 334, 356, 332, 355, 356, 356, 356, 356, 356, 357]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695845359Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"downstairs","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"[1]R generation[0..12]: [186, 184, 183, 190, 182, 190, 190, 190, 190, 190, 190, 191]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695856178Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"downstairs","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695867397Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"downstairs","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"[2]R flush_numbers[0..12]: [347, 341, 334, 356, 332, 355, 356, 356, 356, 356, 356, 357]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695878237Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"downstairs","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"[2]R generation[0..12]: [186, 184, 183, 190, 182, 190, 190, 190, 190, 190, 190, 191]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695889096Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"downstairs","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695900205Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"downstairs","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695912044Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"downstairs","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695922184Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"downstairs","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"Next flush: 358","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695931943Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"downstairs","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695941522Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"":"downstairs","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695951452Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695960481Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"client":"0","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.69597039Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"client":"1","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.69598024Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"client":"2","":"downstairs","session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695990419Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.695999778Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"7617f0b6-9878-4142-a70b-50b2dbc5adc3 is now active with session: 4c201f6a-74cc-4001-b49d-d167be9d6e98","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.696010018Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.696019627Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.696027597Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"session_id":"4c201f6a-74cc-4001-b49d-d167be9d6e98"} {"msg":"Activated sub_volume 7617f0b6-9878-4142-a70b-50b2dbc5adc3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.696036976Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.696045505Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.696378643Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"task":"crutest"} 01/30 Write block 351 len 36864 data: 20 18 16 15 17 16 17 14 18 02/30 Read block 587 len 20480 03/30 Write block 183 len 24576 data: 13 11 14 12 11 15 04/30 Read block 47 len 28672 05/30 Write block 470 len 24576 data: 19 17 13 14 10 11 06/30 Read block 446 len 16384 07/30 Write block 358 len 24576 data: 15 19 19 21 16 16 08/30 Read block 375 len 28672 09/30 Write block 502 len 20480 data: 15 15 18 20 16 10/30 Read block 197 len 32768 11/30 Read block 228 len 28672 12/30 Read block 152 len 32768 13/30 Write block 465 len 20480 data: 18 19 20 20 19 14/30 Write block 592 len 12288 data: 14 12 13 15/30 Read block 156 len 28672 16/30 Write block 227 len 8192 data: 14 18 17/30 Flush 18/30 Flush 19/30 Write block 288 len 16384 data: 10 9 7 9 20/30 Read block 361 len 16384 21/30 Read block 487 len 20480 22/30 Read block 410 len 4096 23/30 Read block 523 len 20480 24/30 Read block 457 len 24576 25/30 Write block 17 len 28672 data: 10 12 12 13 14 16 15 26/30 Read block 461 len 36864 27/30 Read block 498 len 12288 28/30 Read block 429 len 4096 29/30 Write block 180 len 20480 data: 8 9 11 14 12 30/30 Write block 140 len 4096 data: 20 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 5 Sent Done Done false 1002 Acked Write 6 Sent Done Done false 1003 Acked Read 7 Sent Done Done false 1004 Acked Write 6 Sent Done Done false 1005 Acked Read 4 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Write 5 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 7 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Read 5 Sent Done Done false 1021 Acked Read 1 Sent Done Done false 1022 Acked Read 5 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 2 33 Done 1 28 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:39.705176044Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:39.705464745Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1913,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 186 186 186 347 347 347 F T T <--- 1 020-039 184 184 184 341 341 341 F T T <--- 2 040-059 183 183 183 334 334 334 F F F 3 060-079 190 190 190 356 356 356 F F F 4 080-099 182 182 182 332 332 332 F F F 5 100-119 190 190 190 355 355 355 F F F 6 120-139 190 190 190 356 356 356 F F F 7 140-159 190 190 190 356 356 356 F T T <--- 8 160-179 190 190 190 356 356 356 F F F 9 180-199 190 192 192 356 358 358 F T T <--- 10 200-219 190 190 190 356 356 356 F F F 11 220-239 191 192 192 357 358 358 F F F <--- 12 240-259 191 191 191 357 357 357 F F F 13 260-279 190 190 190 356 356 356 F F F 14 280-299 190 190 190 356 356 356 F T T <--- 15 300-319 186 186 186 347 347 347 F F F 16 320-339 190 190 190 355 355 355 F F F 17 340-359 188 192 192 352 358 358 T F F <--- 18 360-379 188 192 192 352 358 358 F F F <--- 19 380-399 186 186 186 345 345 345 F F F 20 400-419 186 186 186 345 345 345 F F F 21 420-439 188 188 188 352 352 352 F F F 22 440-459 186 186 186 345 345 345 F F F 23 460-479 190 192 192 356 358 358 F F F <--- 24 480-499 190 190 190 356 356 356 F F F 25 500-519 191 192 192 357 358 358 F F F <--- 26 520-539 184 184 184 343 343 343 F F F 27 540-559 189 189 189 353 353 353 F F F 28 560-579 188 188 188 349 349 349 F F F 29 580-599 191 192 192 357 358 358 F F F <--- Max gen: 192, Max flush: 358 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 193 193 193 359 359 359 F F F 2 040-059 183 183 183 334 334 334 F F F 3 060-079 190 190 190 356 356 356 F F F 4 080-099 182 182 182 332 332 332 F F F 5 100-119 190 190 190 355 355 355 F F F 6 120-139 190 190 190 356 356 356 F F F 7 140-159 193 193 193 359 359 359 F F F 8 160-179 190 190 190 356 356 356 F F F 9 180-199 193 193 193 359 359 359 F F F 10 200-219 190 190 190 356 356 356 F F F 11 220-239 192 192 192 358 358 358 F F F 12 240-259 191 191 191 357 357 357 F F F 13 260-279 190 190 190 356 356 356 F F F 14 280-299 193 193 193 359 359 359 F F F 15 300-319 186 186 186 347 347 347 F F F 16 320-339 190 190 190 355 355 355 F F F 17 340-359 192 192 192 358 358 358 F F F 18 360-379 192 192 192 358 358 358 F F F 19 380-399 186 186 186 345 345 345 F F F 20 400-419 186 186 186 345 345 345 F F F 21 420-439 188 188 188 352 352 352 F F F 22 440-459 186 186 186 345 345 345 F F F 23 460-479 192 192 192 358 358 358 F F F 24 480-499 190 190 190 356 356 356 F F F 25 500-519 192 192 192 358 358 358 F F F 26 520-539 184 184 184 343 343 343 F F F 27 540-559 189 189 189 353 353 353 F F F 28 560-579 188 188 188 349 349 349 F F F 29 580-599 192 192 192 358 358 358 F F F Max gen: 193, Max flush: 359 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.95368007Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.962377948Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.962676058Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.962704726Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.962713406Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.962725315Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921} {"msg":"Crucible 6321874f-7087-4dab-a30c-be8028de3ba2 has session id: 0b7cf792-7fcc-484f-b41b-9329a476fe8b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.96308571Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"Upstairs opts: Upstairs UUID: 6321874f-7087-4dab-a30c-be8028de3ba2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963114818Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"Crucible stats registered with UUID: 6321874f-7087-4dab-a30c-be8028de3ba2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963126437Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:43.963136387Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963312015Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"io task","client":"0","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963329074Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"io task","client":"1","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963339693Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"io task","client":"2","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963458015Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963475624Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921} {"msg":"6321874f-7087-4dab-a30c-be8028de3ba2 active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963494142Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963503202Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"client":"0","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963513261Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"client":"1","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.96352327Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"client":"2","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.9635333Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"io task","client":"2","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963547729Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"io task","client":"1","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963558148Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"io task","client":"0","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46502)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963629313Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"io task","client":"2","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:46484)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963655401Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"io task","client":"1","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40475)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.963749485Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"io task","client":"0","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.96440949Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"client":"2","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964431189Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"client":"0","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964442738Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"client":"1","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964732918Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964767736Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"[0]R flush_numbers[0..12]: [359, 359, 334, 356, 332, 355, 356, 359, 356, 359, 356, 358]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964824062Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"downstairs","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"[0]R generation[0..12]: [193, 193, 183, 190, 182, 190, 190, 193, 190, 193, 190, 192]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964837131Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"downstairs","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.9648486Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"downstairs","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"[1]R flush_numbers[0..12]: [359, 359, 334, 356, 332, 355, 356, 359, 356, 359, 356, 358]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.9648598Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"downstairs","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"[1]R generation[0..12]: [193, 193, 183, 190, 182, 190, 190, 193, 190, 193, 190, 192]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964870659Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"downstairs","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964881888Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"downstairs","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"[2]R flush_numbers[0..12]: [359, 359, 334, 356, 332, 355, 356, 359, 356, 359, 356, 358]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964892707Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"downstairs","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"[2]R generation[0..12]: [193, 193, 183, 190, 182, 190, 190, 193, 190, 193, 190, 192]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964903477Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"downstairs","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964914886Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"downstairs","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964924545Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"downstairs","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964934594Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"downstairs","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"Next flush: 360","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964944354Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"downstairs","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964954233Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"":"downstairs","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964964292Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964973262Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"client":"0","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.964984621Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"client":"1","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.96499463Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"client":"2","":"downstairs","session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.96500485Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.965014379Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"6321874f-7087-4dab-a30c-be8028de3ba2 is now active with session: 0b7cf792-7fcc-484f-b41b-9329a476fe8b","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.965024638Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.965034338Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.965042177Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"session_id":"0b7cf792-7fcc-484f-b41b-9329a476fe8b"} {"msg":"Activated sub_volume 6321874f-7087-4dab-a30c-be8028de3ba2","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.965051626Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.965059986Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.965432051Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"task":"crutest"} 01/30 Write block 401 len 4096 data: 18 02/30 Write block 482 len 28672 data: 16 17 17 16 15 17 16 03/30 Write block 558 len 24576 data: 17 18 17 17 16 18 04/30 Write block 154 len 28672 data: 12 14 16 11 8 11 10 05/30 Read block 523 len 28672 06/30 Read block 106 len 40960 07/30 Read block 472 len 8192 08/30 Read block 106 len 40960 09/30 Flush 10/30 Write block 563 len 40960 data: 19 17 18 14 16 15 18 18 16 17 11/30 Read block 561 len 40960 12/30 Write block 150 len 16384 data: 21 21 15 12 13/30 Write block 80 len 32768 data: 10 9 7 6 4 6 8 8 14/30 Read block 234 len 24576 15/30 Flush 16/30 Write block 506 len 12288 data: 17 20 17 17/30 Read block 195 len 20480 18/30 Write block 79 len 8192 data: 11 11 19/30 Read block 551 len 36864 20/30 Flush 21/30 Write block 300 len 12288 data: 16 14 13 22/30 Write block 303 len 8192 data: 14 16 23/30 Write block 365 len 32768 data: 17 17 15 16 17 17 14 16 24/30 Write block 362 len 24576 data: 17 17 15 18 18 16 25/30 Write block 203 len 4096 data: 16 26/30 Write block 120 len 8192 data: 12 10 27/30 Write block 119 len 40960 data: 12 13 11 10 9 12 17 20 17 18 28/30 Write block 220 len 20480 data: 17 17 12 9 12 29/30 Write block 230 len 8192 data: 14 16 30/30 Read block 477 len 8192 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Read 7 Done Sent Done false 1005 Acked Read 10 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Write 8 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Read 5 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Write 1 Done Sent Done false 1025 Acked Write 2 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 27 1 28 Done 30 3 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:43.972806729Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:43.973106978Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1921,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 193 193 193 359 359 359 F F F 2 040-059 183 183 183 334 334 334 F F F 3 060-079 194 190 194 362 356 362 F F F <--- 4 080-099 194 182 194 362 332 362 F F F <--- 5 100-119 190 190 190 355 355 355 T F T <--- 6 120-139 190 190 190 356 356 356 T F T <--- 7 140-159 194 193 194 361 359 361 F F F <--- 8 160-179 194 190 194 360 356 360 F F F <--- 9 180-199 193 193 193 359 359 359 F F F 10 200-219 190 190 190 356 356 356 T F T <--- 11 220-239 192 192 192 358 358 358 T F T <--- 12 240-259 191 191 191 357 357 357 F F F 13 260-279 190 190 190 356 356 356 F F F 14 280-299 193 193 193 359 359 359 F F F 15 300-319 186 186 186 347 347 347 T F T <--- 16 320-339 190 190 190 355 355 355 F F F 17 340-359 192 192 192 358 358 358 F F F 18 360-379 192 192 192 358 358 358 T F T <--- 19 380-399 186 186 186 345 345 345 F F F 20 400-419 194 186 194 360 345 360 F T F <--- 21 420-439 188 188 188 352 352 352 F F F 22 440-459 186 186 186 345 345 345 F F F 23 460-479 192 192 192 358 358 358 F F F 24 480-499 194 190 194 360 356 360 F T F <--- 25 500-519 194 192 194 362 358 362 F F F <--- 26 520-539 184 184 184 343 343 343 F F F 27 540-559 194 189 194 360 353 360 F T F <--- 28 560-579 194 188 194 361 349 361 F T F <--- 29 580-599 192 192 192 358 358 358 F F F Max gen: 194, Max flush: 362 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 193 193 193 359 359 359 F F F 2 040-059 183 183 183 334 334 334 F F F 3 060-079 194 194 194 362 362 362 F F F 4 080-099 194 194 194 362 362 362 F F F 5 100-119 195 195 195 363 363 363 F F F 6 120-139 195 195 195 363 363 363 F F F 7 140-159 194 194 194 361 361 361 F F F 8 160-179 194 194 194 360 360 360 F F F 9 180-199 193 193 193 359 359 359 F F F 10 200-219 195 195 195 363 363 363 F F F 11 220-239 195 195 195 363 363 363 F F F 12 240-259 191 191 191 357 357 357 F F F 13 260-279 190 190 190 356 356 356 F F F 14 280-299 193 193 193 359 359 359 F F F 15 300-319 195 195 195 363 363 363 F F F 16 320-339 190 190 190 355 355 355 F F F 17 340-359 192 192 192 358 358 358 F F F 18 360-379 195 195 195 363 363 363 F F F 19 380-399 186 186 186 345 345 345 F F F 20 400-419 194 194 194 360 360 360 F F F 21 420-439 188 188 188 352 352 352 F F F 22 440-459 186 186 186 345 345 345 F F F 23 460-479 192 192 192 358 358 358 F F F 24 480-499 194 194 194 360 360 360 F F F 25 500-519 194 194 194 362 362 362 F F F 26 520-539 184 184 184 343 343 343 F F F 27 540-559 194 194 194 360 360 360 F F F 28 560-579 194 194 194 361 361 361 F F F 29 580-599 192 192 192 358 358 358 F F F Max gen: 195, Max flush: 363 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.225755462Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:48.23298921Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.233053846Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.240079638Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.240325561Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.240350679Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.240362329Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.240374358Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929} {"msg":"Crucible 49558a32-e783-48c8-ad46-acfe0034100a has session id: e9650e66-1a2f-4ca0-91e3-502a44051be3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.240726824Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"Upstairs opts: Upstairs UUID: 49558a32-e783-48c8-ad46-acfe0034100a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.240749512Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"Crucible stats registered with UUID: 49558a32-e783-48c8-ad46-acfe0034100a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.240759812Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:48.240772851Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.240943099Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"io task","client":"1","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.240959778Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"io task","client":"0","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.240970187Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"io task","client":"2","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241090119Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241107578Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929} {"msg":"49558a32-e783-48c8-ad46-acfe0034100a active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241130886Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241140626Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"client":"0","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241150755Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"client":"1","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241160714Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"client":"2","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241172133Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"io task","client":"2","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241182643Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"io task","client":"0","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241195402Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"io task","client":"1","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"ds_connection connected from Ok(127.0.0.1:33437)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241242259Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"io task","client":"0","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"ds_connection connected from Ok(127.0.0.1:57379)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241253518Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"io task","client":"1","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"ds_connection connected from Ok(127.0.0.1:56299)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241264197Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"io task","client":"2","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241884525Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"client":"1","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.241904644Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"client":"2","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242001717Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"client":"0","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242260639Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242284428Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"[0]R flush_numbers[0..12]: [359, 359, 334, 362, 362, 363, 363, 361, 360, 359, 363, 363]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242299167Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"downstairs","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"[0]R generation[0..12]: [193, 193, 183, 194, 194, 195, 195, 194, 194, 193, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242310906Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"downstairs","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242322375Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"downstairs","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"[1]R flush_numbers[0..12]: [359, 359, 334, 362, 362, 363, 363, 361, 360, 359, 363, 363]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242336314Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"downstairs","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"[1]R generation[0..12]: [193, 193, 183, 194, 194, 195, 195, 194, 194, 193, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242347493Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"downstairs","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242358883Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"downstairs","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"[2]R flush_numbers[0..12]: [359, 359, 334, 362, 362, 363, 363, 361, 360, 359, 363, 363]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242369852Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"downstairs","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"[2]R generation[0..12]: [193, 193, 183, 194, 194, 195, 195, 194, 194, 193, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242380701Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"downstairs","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.24239189Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"downstairs","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.24240195Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"downstairs","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242411999Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"downstairs","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"Next flush: 364","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242421788Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"downstairs","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242431548Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"":"downstairs","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242441937Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242451196Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"client":"0","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242461606Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"client":"1","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242471815Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"client":"2","":"downstairs","session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242482344Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242493584Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"49558a32-e783-48c8-ad46-acfe0034100a is now active with session: e9650e66-1a2f-4ca0-91e3-502a44051be3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242503893Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242513412Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242521412Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"session_id":"e9650e66-1a2f-4ca0-91e3-502a44051be3"} {"msg":"Activated sub_volume 49558a32-e783-48c8-ad46-acfe0034100a","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242530831Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.24253929Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.242830591Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"task":"crutest"} 01/30 Write block 311 len 28672 data: 20 19 17 21 21 25 26 02/30 Flush 03/30 Write block 161 len 28672 data: 13 14 14 16 17 15 18 04/30 Write block 114 len 8192 data: 11 9 05/30 Write block 59 len 16384 data: 10 10 11 11 06/30 Read block 50 len 32768 07/30 Read block 17 len 4096 08/30 Read block 193 len 4096 09/30 Write block 501 len 8192 data: 14 16 10/30 Read block 451 len 20480 11/30 Read block 392 len 28672 12/30 Read block 157 len 8192 13/30 Read block 393 len 20480 14/30 Write block 170 len 16384 data: 16 17 14 16 15/30 Write block 166 len 16384 data: 16 19 15 13 16/30 Write block 81 len 36864 data: 10 8 7 5 7 9 9 10 9 17/30 Read block 133 len 20480 18/30 Read block 582 len 20480 19/30 Read block 544 len 24576 20/30 Write block 245 len 24576 data: 15 18 14 10 11 11 21/30 Read block 116 len 40960 22/30 Write block 579 len 12288 data: 16 18 15 23/30 Read block 484 len 40960 24/30 Write block 482 len 36864 data: 17 18 18 17 16 18 17 14 13 25/30 Read block 270 len 28672 26/30 Read block 558 len 4096 27/30 Write block 513 len 32768 data: 13 12 10 11 8 10 12 18 28/30 Write block 383 len 24576 data: 16 14 18 16 13 14 29/30 Write block 413 len 32768 data: 13 14 15 17 16 15 14 15 30/30 Write block 170 len 12288 data: 17 18 15 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 7 Sent Done Done false 1003 Acked Write 2 Sent Done Done false 1004 Acked Write 4 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Write 2 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Read 5 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Write 4 Sent Done Done false 1015 Acked Write 9 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Write 6 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Write 9 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Write 8 Sent Sent Sent false 1027 Acked Write 6 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 4 4 36 Done 0 24 24 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1001)) Some(JobId(1001)) Some(JobId(1001)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:48.250390036Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:48.250692896Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1929,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 193 193 193 359 359 359 F F F 2 040-059 183 183 183 334 334 334 F T T <--- 3 060-079 194 194 194 362 362 362 F T T <--- 4 080-099 194 194 194 362 362 362 F T T <--- 5 100-119 195 195 195 363 363 363 F T T <--- 6 120-139 195 195 195 363 363 363 F F F 7 140-159 194 194 194 361 361 361 F F F 8 160-179 194 194 194 360 360 360 F T T <--- 9 180-199 193 193 193 359 359 359 F F F 10 200-219 195 195 195 363 363 363 F F F 11 220-239 195 195 195 363 363 363 F F F 12 240-259 191 191 191 357 357 357 F T T <--- 13 260-279 190 190 190 356 356 356 F F F 14 280-299 193 193 193 359 359 359 F F F 15 300-319 196 196 196 364 364 364 F F F 16 320-339 190 190 190 355 355 355 F F F 17 340-359 192 192 192 358 358 358 F F F 18 360-379 195 195 195 363 363 363 F F F 19 380-399 186 186 186 345 345 345 F T T <--- 20 400-419 194 194 194 360 360 360 F T T <--- 21 420-439 188 188 188 352 352 352 F T T <--- 22 440-459 186 186 186 345 345 345 F F F 23 460-479 192 192 192 358 358 358 F F F 24 480-499 194 194 194 360 360 360 F T T <--- 25 500-519 194 194 194 362 362 362 F T T <--- 26 520-539 184 184 184 343 343 343 F T T <--- 27 540-559 194 194 194 360 360 360 F F F 28 560-579 194 194 194 361 361 361 F T T <--- 29 580-599 192 192 192 358 358 358 F T T <--- Max gen: 196, Max flush: 364 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 193 193 193 359 359 359 F F F 2 040-059 197 197 197 365 365 365 F F F 3 060-079 197 197 197 365 365 365 F F F 4 080-099 197 197 197 365 365 365 F F F 5 100-119 197 197 197 365 365 365 F F F 6 120-139 195 195 195 363 363 363 F F F 7 140-159 194 194 194 361 361 361 F F F 8 160-179 197 197 197 365 365 365 F F F 9 180-199 193 193 193 359 359 359 F F F 10 200-219 195 195 195 363 363 363 F F F 11 220-239 195 195 195 363 363 363 F F F 12 240-259 197 197 197 365 365 365 F F F 13 260-279 190 190 190 356 356 356 F F F 14 280-299 193 193 193 359 359 359 F F F 15 300-319 196 196 196 364 364 364 F F F 16 320-339 190 190 190 355 355 355 F F F 17 340-359 192 192 192 358 358 358 F F F 18 360-379 195 195 195 363 363 363 F F F 19 380-399 197 197 197 365 365 365 F F F 20 400-419 197 197 197 365 365 365 F F F 21 420-439 197 197 197 365 365 365 F F F 22 440-459 186 186 186 345 345 345 F F F 23 460-479 192 192 192 358 358 358 F F F 24 480-499 197 197 197 365 365 365 F F F 25 500-519 197 197 197 365 365 365 F F F 26 520-539 197 197 197 365 365 365 F F F 27 540-559 194 194 194 360 360 360 F F F 28 560-579 197 197 197 365 365 365 F F F 29 580-599 197 197 197 365 365 365 F F F Max gen: 197, Max flush: 365 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.507801396Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.516203665Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.516487835Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"a1ad264a2bdf567f85c6d702192e1961ae046a2c\",\n git_commit_timestamp: \"2026-07-28T04:22:56.000000000Z\",\n git_branch: \"renovate/rust-1.x\",\n rustc_semver: \"1.97.1\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"8bab26f4f68e0e26f0bb7960be334d5b520ea452\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.516514153Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.516523393Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.516535322Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937} {"msg":"Crucible 37f2a86f-8560-4aad-ae99-fbc5312046af has session id: 71d60db7-fbe4-49f2-9efc-9404cb86c4c7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.516924576Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"Upstairs opts: Upstairs UUID: 37f2a86f-8560-4aad-ae99-fbc5312046af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.516954504Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"Crucible stats registered with UUID: 37f2a86f-8560-4aad-ae99-fbc5312046af","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.516967993Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:52.516977862Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.517131631Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"io task","client":"1","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.51714867Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"io task","client":"0","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.51715937Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.51730328Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.517328488Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937} {"msg":"37f2a86f-8560-4aad-ae99-fbc5312046af active request set","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.517345797Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.517354916Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"client":"0","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.517365126Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"client":"1","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.517375015Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"client":"2","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.517384964Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.517401363Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"io task","client":"1","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.517411792Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"io task","client":"0","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:40735)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.517457899Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"io task","client":"1","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:59742)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.517483948Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"io task","client":"2","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:41885)","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.517497407Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"io task","client":"0","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518255845Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"client":"1","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518299402Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"client":"0","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518325Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"client":"2","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518563444Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518603721Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"[0]R flush_numbers[0..12]: [359, 359, 365, 365, 365, 365, 363, 361, 365, 359, 363, 363]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518654128Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"downstairs","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"[0]R generation[0..12]: [193, 193, 197, 197, 197, 197, 195, 194, 197, 193, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518666677Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"downstairs","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518678026Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"downstairs","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"[1]R flush_numbers[0..12]: [359, 359, 365, 365, 365, 365, 363, 361, 365, 359, 363, 363]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518689036Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"downstairs","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"[1]R generation[0..12]: [193, 193, 197, 197, 197, 197, 195, 194, 197, 193, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518699945Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"downstairs","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518711204Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"downstairs","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"[2]R flush_numbers[0..12]: [359, 359, 365, 365, 365, 365, 363, 361, 365, 359, 363, 363]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518722083Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"downstairs","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"[2]R generation[0..12]: [193, 193, 197, 197, 197, 197, 195, 194, 197, 193, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518732803Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"downstairs","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518746582Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"downstairs","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518756361Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"downstairs","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.5187664Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"downstairs","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"Next flush: 366","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.5187765Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"downstairs","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518786309Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"":"downstairs","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518796438Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518805558Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"client":"0","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518815547Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"client":"1","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518825496Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"client":"2","":"downstairs","session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518835776Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518845115Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"37f2a86f-8560-4aad-ae99-fbc5312046af is now active with session: 71d60db7-fbe4-49f2-9efc-9404cb86c4c7","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518855414Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518864884Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518872713Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"session_id":"71d60db7-fbe4-49f2-9efc-9404cb86c4c7"} {"msg":"Activated sub_volume 37f2a86f-8560-4aad-ae99-fbc5312046af","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518882082Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937} {"msg":"Wait for a query_work_queue command to finish before sending IO","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.518890642Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"task":"crutest"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair/test_repair_verify.data" {"msg":"Run Repair workload","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.519228499Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"task":"crutest"} 01/30 Write block 115 len 24576 data: 10 10 10 12 13 14 02/30 Write block 548 len 4096 data: 18 03/30 Read block 458 len 8192 04/30 Read block 557 len 24576 05/30 Write block 122 len 28672 data: 11 10 13 18 21 18 19 06/30 Read block 375 len 36864 07/30 Read block 562 len 24576 08/30 Flush 09/30 Write block 552 len 12288 data: 19 20 19 10/30 Write block 430 len 8192 data: 13 16 11/30 Read block 564 len 32768 12/30 Write block 392 len 20480 data: 17 15 17 17 17 13/30 Read block 288 len 12288 14/30 Flush 15/30 Write block 77 len 12288 data: 13 13 12 16/30 Read block 445 len 24576 17/30 Write block 444 len 32768 data: 10 9 7 8 10 12 11 12 18/30 Read block 172 len 16384 19/30 Write block 87 len 24576 data: 10 11 10 10 10 14 20/30 Read block 466 len 36864 21/30 Write block 183 len 12288 data: 15 13 15 22/30 Read block 454 len 28672 23/30 Read block 444 len 8192 24/30 Write block 314 len 8192 data: 22 22 25/30 Read block 350 len 4096 26/30 Flush 27/30 Write block 398 len 36864 data: 20 21 19 19 18 16 13 14 13 28/30 Read block 9 len 24576 29/30 Write block 524 len 16384 data: 16 17 16 15 30/30 Write block 478 len 8192 data: 15 14 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 3 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Read 8 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 6 Done Sent Done false 1016 Acked Write 8 Done Sent Done false 1017 Acked Read 4 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Read 9 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Read 2 Done Sent Done false 1023 Acked Write 2 Done Sent Done false 1024 Acked Read 1 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 9 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 23 2 27 Done 28 7 28 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-07-28T04:45:52.528071297Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937} {"msg":"Wrote out file \"/tmp/test_repair/test_repair_verify.data\"","v":0,"name":"crucible","level":30,"time":"2026-07-28T04:45:52.528374657Z","hostname":"w-01KYKG6MPV2KVEDG1YGBGPQYK4","pid":1937,"task":"crutest"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 193 193 193 359 359 359 F F F 2 040-059 197 197 197 365 365 365 F F F 3 060-079 198 197 198 368 365 368 F F F <--- 4 080-099 198 197 198 368 365 368 F F F <--- 5 100-119 198 197 198 366 365 366 F T F <--- 6 120-139 198 195 198 366 363 366 F T F <--- 7 140-159 194 194 194 361 361 361 F F F 8 160-179 197 197 197 365 365 365 F F F 9 180-199 198 193 198 368 359 368 F F F <--- 10 200-219 195 195 195 363 363 363 F F F 11 220-239 195 195 195 363 363 363 F F F 12 240-259 197 197 197 365 365 365 F F F 13 260-279 190 190 190 356 356 356 F F F 14 280-299 193 193 193 359 359 359 F F F 15 300-319 198 196 198 368 364 368 F F F <--- 16 320-339 190 190 190 355 355 355 F F F 17 340-359 192 192 192 358 358 358 F F F 18 360-379 195 195 195 363 363 363 F F F 19 380-399 198 197 198 367 365 367 T F T <--- 20 400-419 197 197 197 365 365 365 T F T <--- 21 420-439 198 197 198 367 365 367 F F F <--- 22 440-459 198 186 198 368 345 368 F F F <--- 23 460-479 192 192 192 358 358 358 T F T <--- 24 480-499 197 197 197 365 365 365 F F F 25 500-519 197 197 197 365 365 365 F F F 26 520-539 197 197 197 365 365 365 T F T <--- 27 540-559 198 194 198 367 360 367 F T F <--- 28 560-579 197 197 197 365 365 365 F F F 29 580-599 197 197 197 365 365 365 F F F Max gen: 198, Max flush: 368 Error: Difference in extent metadata found! Stack backtrace: 0: ::msg::<&str> at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 359 359 359 F F F 1 020-039 193 193 193 359 359 359 F F F 2 040-059 197 197 197 365 365 365 F F F 3 060-079 198 198 198 368 368 368 F F F 4 080-099 198 198 198 368 368 368 F F F 5 100-119 198 198 198 366 366 366 F F F 6 120-139 198 198 198 366 366 366 F F F 7 140-159 194 194 194 361 361 361 F F F 8 160-179 197 197 197 365 365 365 F F F 9 180-199 198 198 198 368 368 368 F F F 10 200-219 195 195 195 363 363 363 F F F 11 220-239 195 195 195 363 363 363 F F F 12 240-259 197 197 197 365 365 365 F F F 13 260-279 190 190 190 356 356 356 F F F 14 280-299 193 193 193 359 359 359 F F F 15 300-319 198 198 198 368 368 368 F F F 16 320-339 190 190 190 355 355 355 F F F 17 340-359 192 192 192 358 358 358 F F F 18 360-379 195 195 195 363 363 363 F F F 19 380-399 199 199 199 369 369 369 F F F 20 400-419 199 199 199 369 369 369 F F F 21 420-439 198 198 198 367 367 367 F F F 22 440-459 198 198 198 368 368 368 F F F 23 460-479 199 199 199 369 369 369 F F F 24 480-499 197 197 197 365 365 365 F F F 25 500-519 197 197 197 365 365 365 F F F 26 520-539 199 199 199 369 369 369 F F F 27 540-559 198 198 198 367 367 367 F F F 28 560-579 197 197 197 365 365 365 F F F 29 580-599 197 197 197 365 365 365 F F F Max gen: 199, Max flush: 369 7:27 Test duration Test completed real 7:27.132309049 user 25.211392433 sys 14.298397449 trap 0.017483535 tflt 0.078678501 dflt 0.097053930 kflt 0.003759946 lock 11:47:33.471329425 slp 35:57.232598136 lat 2.240311826 stop 0.020185125